aboutsummaryrefslogtreecommitdiff
path: root/ext/wasm/api
diff options
context:
space:
mode:
Diffstat (limited to 'ext/wasm/api')
-rw-r--r--ext/wasm/api/extern-post-js.js45
-rw-r--r--ext/wasm/api/pre-js.js44
-rw-r--r--ext/wasm/api/sqlite3-api-opfs.js14
-rw-r--r--ext/wasm/api/sqlite3-api-prologue.js16
-rw-r--r--ext/wasm/api/sqlite3-opfs-async-proxy.js730
-rw-r--r--ext/wasm/api/sqlite3-worker1-promiser.js259
-rw-r--r--ext/wasm/api/sqlite3-worker1.js53
7 files changed, 1149 insertions, 12 deletions
diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js
index 7dba03b3a..5f5f72d4b 100644
--- a/ext/wasm/api/extern-post-js.js
+++ b/ext/wasm/api/extern-post-js.js
@@ -16,6 +16,27 @@
if(!originalInit){
throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build.");
}
+ /**
+ We need to add some state which our custom Module.locateFile()
+ can see, but an Emscripten limitation currently prevents us from
+ attaching it to the sqlite3InitModule function object:
+
+ https://github.com/emscripten-core/emscripten/issues/18071
+
+ The only current workaround is to temporarily stash this state
+ into the global scope and delete it when sqlite3InitModule()
+ is called.
+ */
+ const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{
+ moduleScript: self?.document?.currentScript,
+ isWorker: (!self.document && self.window !== self),
+ location: self.location,
+ urlParams: new URL(self.location.href).searchParams
+ });
+ if(initModuleState.urlParams.has('sqlite3.dir')){
+ initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/';
+ };
+
self.sqlite3InitModule = (...args)=>{
//console.warn("Using replaced sqlite3InitModule()",self.location);
return originalInit(...args).then((EmscriptenModule)=>{
@@ -32,6 +53,8 @@
Emscripten details. */
return EmscriptenModule;
}
+ EmscriptenModule.sqlite3.scriptInfo = initModuleState;
+ //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo);
const f = EmscriptenModule.sqlite3.asyncPostInit;
delete EmscriptenModule.sqlite3.asyncPostInit;
return f();
@@ -41,13 +64,19 @@
});
};
self.sqlite3InitModule.ready = originalInit.ready;
- //console.warn("Replaced sqlite3InitModule()");
-})();
-if(0){
- console.warn("self.location.href =",self.location.href);
- if('undefined' !== typeof document){
- console.warn("document.currentScript.src =",
- document?.currentScript?.src);
+ if(self.sqlite3InitModuleState.moduleScript){
+ const sim = self.sqlite3InitModuleState;
+ let src = sim.moduleScript.src.split('/');
+ src.pop();
+ sim.scriptDir = src.join('/') + '/';
}
-}
+ if(0){
+ console.warn("Replaced sqlite3InitModule()");
+ console.warn("self.location.href =",self.location.href);
+ if('undefined' !== typeof document){
+ console.warn("document.currentScript.src =",
+ document?.currentScript?.src);
+ }
+ }
+})();
diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js
index b6630416d..c07d0373c 100644
--- a/ext/wasm/api/pre-js.js
+++ b/ext/wasm/api/pre-js.js
@@ -4,9 +4,49 @@
This file is intended to be prepended to the sqlite3.js build using
Emscripten's --pre-js=THIS_FILE flag (or equivalent).
*/
+
+// See notes in extern-post-js.js
+const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null);
+delete self.sqlite3InitModuleState;
+
+/**
+ This custom locateFile() tries to figure out where to load `path`
+ from. The intent is to provide a way for foo/bar/X.js loaded from a
+ Worker constructor or importScripts() to be able to resolve
+ foo/bar/X.wasm (in the latter case, with some help):
+
+ 1) If URL param named the same as `path` is set, it is returned.
+
+ 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path)
+ is returned (note that it's assumed to end with '/').
+
+ 3) If this code is running in the main UI thread AND it was loaded
+ from a SCRIPT tag, the directory part of that URL is used
+ as the prefix. (This form of resolution unfortunately does not
+ function for scripts loaded via importScripts().)
+
+ 4) If none of the above apply, (prefix+path) is returned.
+*/
Module['locateFile'] = function(path, prefix) {
- return prefix + path;
-};
+ let theFile;
+ const up = this.urlParams;
+ if(0){
+ console.warn("locateFile(",arguments[0], ',', arguments[1],")",
+ 'self.location =',self.location,
+ 'sqlite3InitModuleState.scriptDir =',this.scriptDir,
+ 'up.entries() =',Array.from(up.entries()));
+ }
+ if(up.has(path)){
+ theFile = up.get(path);
+ }else if(this.sqlite3Dir){
+ theFile = this.sqlite3Dir + path;
+ }else if(this.scriptDir){
+ theFile = this.scriptDir + path;
+ }else{
+ theFile = prefix + path;
+ }
+ return theFile;
+}.bind(sqlite3InitModuleState);
/**
Bug warning: this xInstantiateWasm bit must remain disabled
diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js
index 98defe4fb..9f5302454 100644
--- a/ext/wasm/api/sqlite3-api-opfs.js
+++ b/ext/wasm/api/sqlite3-api-opfs.js
@@ -98,6 +98,9 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
options.proxyUri = callee.defaultProxyUri;
}
+ if('function' === typeof options.proxyUri){
+ options.proxyUri = options.proxyUri();
+ }
const thePromise = new Promise(function(promiseResolve, promiseReject_){
const loggers = {
0:console.error.bind(console),
@@ -1092,9 +1095,18 @@ installOpfsVfs.defaultProxyUri =
"sqlite3-opfs-async-proxy.js";
//console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri);
self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
+ if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){
+ return;
+ }
try{
+ let proxyJs = installOpfsVfs.defaultProxyUri;
+ if(sqlite3.scriptInfo.sqlite3Dir){
+ installOpfsVfs.defaultProxyUri =
+ sqlite3.scriptInfo.sqlite3Dir + proxyJs;
+ //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri);
+ }
return installOpfsVfs().catch((e)=>{
- console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e);
+ console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message);
});
}catch(e){
console.error("installOpfsVfs() exception:",e);
diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js
index f8c0f024e..3980e0ad4 100644
--- a/ext/wasm/api/sqlite3-api-prologue.js
+++ b/ext/wasm/api/sqlite3-api-prologue.js
@@ -1334,7 +1334,20 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
//while(lip.length) p = p.then(lip.shift());
//return p.then(()=>sqlite3);
return Promise.all(lip).then(()=>sqlite3);
- }
+ },
+ /**
+ scriptInfo ideally gets injected into this object by the
+ infrastructure which assembles the JS/WASM module. It contains
+ state which must be collected before sqlite3ApiBootstrap() can
+ be declared. It is not necessarily available to any
+ sqlite3ApiBootstrap.initializers but "should" be in place (if
+ it's added at all) by the time that
+ sqlite3ApiBootstrap.initializersAsync is processed.
+
+ This state is not part of the public API, only intended for use
+ with the sqlite3 API bootstrapping and wasm-loading process.
+ */
+ scriptInfo: undefined
};
try{
sqlite3ApiBootstrap.initializers.forEach((f)=>{
@@ -1410,3 +1423,4 @@ self.sqlite3ApiBootstrap.defaultConfig = Object.create(null);
value which will be stored here.
*/
self.sqlite3ApiBootstrap.sqlite3 = undefined;
+
diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js
new file mode 100644
index 000000000..5b29aa3ea
--- /dev/null
+++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js
@@ -0,0 +1,730 @@
+/*
+ 2022-09-16
+
+ The author disclaims copyright to this source code. In place of a
+ legal notice, here is a blessing:
+
+ * May you do good and not evil.
+ * May you find forgiveness for yourself and forgive others.
+ * May you share freely, never taking more than you give.
+
+ ***********************************************************************
+
+ An Worker which manages asynchronous OPFS handles on behalf of a
+ synchronous API which controls it via a combination of Worker
+ messages, SharedArrayBuffer, and Atomics. It is the asynchronous
+ counterpart of the API defined in sqlite3-api-opfs.js.
+
+ Highly indebted to:
+
+ https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
+
+ for demonstrating how to use the OPFS APIs.
+
+ This file is to be loaded as a Worker. It does not have any direct
+ access to the sqlite3 JS/WASM bits, so any bits which it needs (most
+ notably SQLITE_xxx integer codes) have to be imported into it via an
+ initialization process.
+*/
+'use strict';
+const toss = function(...args){throw new Error(args.join(' '))};
+if(self.window === self){
+ toss("This code cannot run from the main thread.",
+ "Load it as a Worker from a separate Worker.");
+}else if(!navigator.storage.getDirectory){
+ toss("This API requires navigator.storage.getDirectory.");
+}
+
+/**
+ Will hold state copied to this object from the syncronous side of
+ this API.
+*/
+const state = Object.create(null);
+/**
+ verbose:
+
+ 0 = no logging output
+ 1 = only errors
+ 2 = warnings and errors
+ 3 = debug, warnings, and errors
+*/
+state.verbose = 2;
+
+const loggers = {
+ 0:console.error.bind(console),
+ 1:console.warn.bind(console),
+ 2:console.log.bind(console)
+};
+const logImpl = (level,...args)=>{
+ if(state.verbose>level) loggers[level]("OPFS asyncer:",...args);
+};
+const log = (...args)=>logImpl(2, ...args);
+const warn = (...args)=>logImpl(1, ...args);
+const error = (...args)=>logImpl(0, ...args);
+const metrics = Object.create(null);
+metrics.reset = ()=>{
+ let k;
+ const r = (m)=>(m.count = m.time = m.wait = 0);
+ for(k in state.opIds){
+ r(metrics[k] = Object.create(null));
+ }
+ let s = metrics.s11n = Object.create(null);
+ s = s.serialize = Object.create(null);
+ s.count = s.time = 0;
+ s = metrics.s11n.deserialize = Object.create(null);
+ s.count = s.time = 0;
+};
+metrics.dump = ()=>{
+ let k, n = 0, t = 0, w = 0;
+ for(k in state.opIds){
+ const m = metrics[k];
+ n += m.count;
+ t += m.time;
+ w += m.wait;
+ m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
+ }
+ console.log(self.location.href,
+ "metrics for",self.location.href,":\n",
+ metrics,
+ "\nTotal of",n,"op(s) for",t,"ms",
+ "approx",w,"ms spent waiting on OPFS APIs.");
+ console.log("Serialization metrics:",metrics.s11n);
+};
+
+/**
+ Map of sqlite3_file pointers (integers) to metadata related to a
+ given OPFS file handles. The pointers are, in this side of the
+ interface, opaque file handle IDs provided by the synchronous
+ part of this constellation. Each value is an object with a structure
+ demonstrated in the xOpen() impl.
+*/
+const __openFiles = Object.create(null);
+
+/**
+ Expects an OPFS file path. It gets resolved, such that ".."
+ components are properly expanded, and returned. If the 2nd arg is
+ true, the result is returned as an array of path elements, else an
+ absolute path string is returned.
+*/
+const getResolvedPath = function(filename,splitIt){
+ const p = new URL(
+ filename, 'file://irrelevant'
+ ).pathname;
+ return splitIt ? p.split('/').filter((v)=>!!v) : p;
+};
+
+/**
+ Takes the absolute path to a filesystem element. Returns an array
+ of [handleOfContainingDir, filename]. If the 2nd argument is truthy
+ then each directory element leading to the file is created along
+ the way. Throws if any creation or resolution fails.
+*/
+const getDirForFilename = async function f(absFilename, createDirs = false){
+ const path = getResolvedPath(absFilename, true);
+ const filename = path.pop();
+ let dh = state.rootDir;
+ for(const dirName of path){
+ if(dirName){
+ dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
+ }
+ }
+ return [dh, filename];
+};
+
+/**
+ Returns the sync access handle associated with the given file
+ handle object (which must be a valid handle object), lazily opening
+ it if needed.
+
+ In order to help alleviate cross-tab contention for a dabase,
+ if an exception is thrown while acquiring the handle, this routine
+ will wait briefly and try again, up to 3 times. If acquisition
+ still fails at that point it will give up and propagate the
+ exception.
+*/
+const getSyncHandle = async (fh)=>{
+ if(!fh.syncHandle){
+ const t = performance.now();
+ log("Acquiring sync handle for",fh.filenameAbs);
+ const maxTries = 3;
+ let i = 1, ms = 300;
+ for(; true; ms *= ++i){
+ try {
+ //if(1===i) toss("Just testing.");
+ //TODO? A config option which tells it to throw here
+ //randomly every now and then, for testing purposes.
+ fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
+ break;
+ }catch(e){
+ if(i === maxTries){
+ toss("Error getting sync handle.",maxTries,
+ "attempts failed. ",fh.filenameAbs, ":", e.message);
+ throw e;
+ }
+ warn("Error getting sync handle. Waiting",ms,
+ "ms and trying again.",fh.filenameAbs,e);
+ Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
+ }
+ }
+ log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
+ }
+ return fh.syncHandle;
+};
+
+/**
+ If the given file-holding object has a sync handle attached to it,
+ that handle is remove and asynchronously closed. Though it may
+ sound sensible to continue work as soon as the close() returns
+ (noting that it's asynchronous), doing so can cause operations
+ performed soon afterwards, e.g. a call to getSyncHandle() to fail
+ because they may happen out of order from the close(). OPFS does
+ not guaranty that the actual order of operations is retained in
+ such cases. i.e. always "await" on the result of this function.
+*/
+const closeSyncHandle = async (fh)=>{
+ if(fh.syncHandle){
+ log("Closing sync handle for",fh.filenameAbs);
+ const h = fh.syncHandle;
+ delete fh.syncHandle;
+ return h.close();
+ }
+};
+
+/**
+ Stores the given value at state.sabOPView[state.opIds.rc] and then
+ Atomics.notify()'s it.
+*/
+const storeAndNotify = (opName, value)=>{
+ log(opName+"() => notify(",state.opIds.rc,",",value,")");
+ Atomics.store(state.sabOPView, state.opIds.rc, value);
+ Atomics.notify(state.sabOPView, state.opIds.rc);
+};
+
+/**
+ Throws if fh is a file-holding object which is flagged as read-only.
+*/
+const affirmNotRO = function(opName,fh){
+ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
+};
+
+/**
+ We track 2 different timers: the "metrics" timer records how much
+ time we spend performing work. The "wait" timer records how much
+ time we spend waiting on the underlying OPFS timer. See the calls
+ to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd()
+ throughout this file to see how they're used.
+*/
+const __mTimer = Object.create(null);
+__mTimer.op = undefined;
+__mTimer.start = undefined;
+const mTimeStart = (op)=>{
+ __mTimer.start = performance.now();
+ __mTimer.op = op;
+ //metrics[op] || toss("Maintenance required: missing metrics for",op);
+ ++metrics[op].count;
+};
+const mTimeEnd = ()=>(
+ metrics[__mTimer.op].time += performance.now() - __mTimer.start
+);
+const __wTimer = Object.create(null);
+__wTimer.op = undefined;
+__wTimer.start = undefined;
+const wTimeStart = (op)=>{
+ __wTimer.start = performance.now();
+ __wTimer.op = op;
+ //metrics[op] || toss("Maintenance required: missing metrics for",op);
+};
+const wTimeEnd = ()=>(
+ metrics[__wTimer.op].wait += performance.now() - __wTimer.start
+);
+
+/**
+ Gets set to true by the 'opfs-async-shutdown' command to quit the
+ wait loop. This is only intended for debugging purposes: we cannot
+ inspect this file's state while the tight waitLoop() is running and
+ need a way to stop that loop for introspection purposes.
+*/
+let flagAsyncShutdown = false;
+
+
+/**
+ Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
+ methods, as well as helpers like mkdir(). Maintenance reminder:
+ members are in alphabetical order to simplify finding them.
+*/
+const vfsAsyncImpls = {
+ 'opfs-async-metrics': async ()=>{
+ mTimeStart('opfs-async-metrics');
+ metrics.dump();
+ storeAndNotify('opfs-async-metrics', 0);
+ mTimeEnd();
+ },
+ 'opfs-async-shutdown': async ()=>{
+ flagAsyncShutdown = true;
+ storeAndNotify('opfs-async-shutdown', 0);
+ },
+ mkdir: async (dirname)=>{
+ mTimeStart('mkdir');
+ let rc = 0;
+ wTimeStart('mkdir');
+ try {
+ await getDirForFilename(dirname+"/filepart", true);
+ }catch(e){
+ state.s11n.storeException(2,e);
+ rc = state.sq3Codes.SQLITE_IOERR;
+ }finally{
+ wTimeEnd();
+ }
+ storeAndNotify('mkdir', rc);
+ mTimeEnd();
+ },
+ xAccess: async (filename)=>{
+ mTimeStart('xAccess');
+ /* OPFS cannot support the full range of xAccess() queries sqlite3
+ calls for. We can essentially just tell if the file is
+ accessible, but if it is it's automatically writable (unless
+ it's locked, which we cannot(?) know without trying to open
+ it). OPFS does not have the notion of read-only.
+
+ The return semantics of this function differ from sqlite3's
+ xAccess semantics because we are limited in what we can
+ communicate back to our synchronous communication partner: 0 =
+ accessible, non-0 means not accessible.
+ */
+ let rc = 0;
+ wTimeStart('xAccess');
+ try{
+ const [dh, fn] = await getDirForFilename(filename);
+ await dh.getFileHandle(fn);
+ }catch(e){
+ state.s11n.storeException(2,e);
+ rc = state.sq3Codes.SQLITE_IOERR;
+ }finally{
+ wTimeEnd();
+ }
+ storeAndNotify('xAccess', rc);
+ mTimeEnd();
+ },
+ xClose: async function(fid){
+ const opName = 'xClose';
+ mTimeStart(opName);
+ const fh = __openFiles[fid];
+ let rc = 0;
+ wTimeStart('xClose');
+ if(fh){
+ delete __openFiles[fid];
+ await closeSyncHandle(fh);
+ if(fh.deleteOnClose){
+ try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
+ catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
+ }
+ }else{
+ state.s11n.serialize();
+ rc = state.sq3Codes.SQLITE_NOTFOUND;
+ }
+ wTimeEnd();
+ storeAndNotify(opName, rc);
+ mTimeEnd();
+ },
+ xDelete: async function(...args){
+ mTimeStart('xDelete');
+ const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
+ storeAndNotify('xDelete', rc);
+ mTimeEnd();
+ },
+ xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){
+ /* The syncDir flag is, for purposes of the VFS API's semantics,
+ ignored here. However, if it has the value 0x1234 then: after
+ deleting the given file, recursively try to delete any empty
+ directories left behind in its wake (ignoring any errors and
+ stopping at the first failure).
+
+ That said: we don't know for sure that removeEntry() fails if
+ the dir is not empty because the API is not documented. It has,
+ however, a "recursive" flag which defaults to false, so
+ presumably it will fail if the dir is not empty and that flag
+ is false.
+ */
+ let rc = 0;
+ wTimeStart('xDelete');
+ try {
+ while(filename){
+ const [hDir, filenamePart] = await getDirForFilename(filename, false);
+ if(!filenamePart) break;
+ await hDir.removeEntry(filenamePart, {recursive});
+ if(0x1234 !== syncDir) break;
+ filename = getResolvedPath(filename, true);
+ filename.pop();
+ filename = filename.join('/');
+ }
+ }catch(e){
+ state.s11n.storeException(2,e);
+ rc = state.sq3Codes.SQLITE_IOERR_DELETE;
+ }
+ wTimeEnd();
+ return rc;
+ },
+ xFileSize: async function(fid){
+ mTimeStart('xFileSize');
+ const fh = __openFiles[fid];
+ let sz;
+ wTimeStart('xFileSize');
+ try{
+ sz = await (await getSyncHandle(fh)).getSize();
+ state.s11n.serialize(Number(sz));
+ sz = 0;
+ }catch(e){
+ state.s11n.storeException(2,e);
+ sz = state.sq3Codes.SQLITE_IOERR;
+ }
+ wTimeEnd();
+ storeAndNotify('xFileSize', sz);
+ mTimeEnd();
+ },
+ xLock: async function(fid,lockType){
+ mTimeStart('xLock');
+ const fh = __openFiles[fid];
+ let rc = 0;
+ if( !fh.syncHandle ){
+ wTimeStart('xLock');
+ try { await getSyncHandle(fh) }
+ catch(e){
+ state.s11n.storeException(1,e);
+ rc = state.sq3Codes.SQLITE_IOERR;
+ }
+ wTimeEnd();
+ }
+ storeAndNotify('xLock',rc);
+ mTimeEnd();
+ },
+ xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){
+ const opName = 'xOpen';
+ mTimeStart(opName);
+ const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
+ const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
+ wTimeStart('xOpen');
+ try{
+ let hDir, filenamePart;
+ try {
+ [hDir, filenamePart] = await getDirForFilename(filename, !!create);
+ }catch(e){
+ storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
+ mTimeEnd();
+ wTimeEnd();
+ return;
+ }
+ const hFile = await hDir.getFileHandle(filenamePart, {create});
+ /**
+ wa-sqlite, at this point, grabs a SyncAccessHandle and
+ assigns it to the syncHandle prop of the file state
+ object, but only for certain cases and it's unclear why it
+ places that limitation on it.
+ */
+ wTimeEnd();
+ __openFiles[fid] = Object.assign(Object.create(null),{
+ filenameAbs: filename,
+ filenamePart: filenamePart,
+ dirHandle: hDir,
+ fileHandle: hFile,
+ sabView: state.sabFileBufView,
+ readOnly: create
+ ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
+ deleteOnClose: deleteOnClose
+ });
+ storeAndNotify(opName, 0);
+ }catch(e){
+ wTimeEnd();
+ error(opName,e);
+ state.s11n.storeException(1,e);
+ storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
+ }
+ mTimeEnd();
+ },
+ xRead: async function(fid,n,offset){
+ mTimeStart('xRead');
+ let rc = 0, nRead;
+ const fh = __openFiles[fid];
+ try{
+ wTimeStart('xRead');
+ nRead = (await getSyncHandle(fh)).read(
+ fh.sabView.subarray(0, n),
+ {at: Number(offset)}
+ );
+ wTimeEnd();
+ if(nRead < n){/* Zero-fill remaining bytes */
+ fh.sabView.fill(0, nRead, n);
+ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
+ }
+ }catch(e){
+ if(undefined===nRead) wTimeEnd();
+ error("xRead() failed",e,fh);
+ state.s11n.storeException(1,e);
+ rc = state.sq3Codes.SQLITE_IOERR_READ;
+ }
+ storeAndNotify('xRead',rc);
+ mTimeEnd();
+ },
+ xSync: async function(fid,flags/*ignored*/){
+ mTimeStart('xSync');
+ const fh = __openFiles[fid];
+ let rc = 0;
+ if(!fh.readOnly && fh.syncHandle){
+ try {
+ wTimeStart('xSync');
+ await fh.syncHandle.flush();
+ }catch(e){
+ state.s11n.storeException(2,e);
+ }
+ wTimeEnd();
+ }
+ storeAndNotify('xSync',rc);
+ mTimeEnd();
+ },
+ xTruncate: async function(fid,size){
+ mTimeStart('xTruncate');
+ let rc = 0;
+ const fh = __openFiles[fid];
+ wTimeStart('xTruncate');
+ try{
+ affirmNotRO('xTruncate', fh);
+ await (await getSyncHandle(fh)).truncate(size);
+ }catch(e){
+ error("xTruncate():",e,fh);
+ state.s11n.storeException(2,e);
+ rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
+ }
+ wTimeEnd();
+ storeAndNotify('xTruncate',rc);
+ mTimeEnd();
+ },
+ xUnlock: async function(fid,lockType){
+ mTimeStart('xUnlock');
+ let rc = 0;
+ const fh = __openFiles[fid];
+ if( state.sq3Codes.SQLITE_LOCK_NONE===lockType
+ && fh.syncHandle ){
+ wTimeStart('xUnlock');
+ try { await closeSyncHandle(fh) }
+ catch(e){
+ state.s11n.storeException(1,e);
+ rc = state.sq3Codes.SQLITE_IOERR;
+ }
+ wTimeEnd();
+ }
+ storeAndNotify('xUnlock',rc);
+ mTimeEnd();
+ },
+ xWrite: async function(fid,n,offset){
+ mTimeStart('xWrite');
+ let rc;
+ wTimeStart('xWrite');
+ try{
+ const fh = __openFiles[fid];
+ affirmNotRO('xWrite', fh);
+ rc = (
+ n === (await getSyncHandle(fh))
+ .write(fh.sabView.subarray(0, n),
+ {at: Number(offset)})
+ ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
+ }catch(e){
+ error("xWrite():",e,fh);
+ state.s11n.storeException(1,e);
+ rc = state.sq3Codes.SQLITE_IOERR_WRITE;
+ }
+ wTimeEnd();
+ storeAndNotify('xWrite',rc);
+ mTimeEnd();
+ }
+}/*vfsAsyncImpls*/;
+
+const initS11n = ()=>{
+ /**
+ ACHTUNG: this code is 100% duplicated in the other half of this
+ proxy! The documentation is maintained in the "synchronous half".
+ */
+ if(state.s11n) return state.s11n;
+ const textDecoder = new TextDecoder(),
+ textEncoder = new TextEncoder('utf-8'),
+ viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
+ viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
+ state.s11n = Object.create(null);
+ const TypeIds = Object.create(null);
+ TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
+ TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
+ TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
+ TypeIds.string = { id: 4 };
+ const getTypeId = (v)=>(
+ TypeIds[typeof v]
+ || toss("Maintenance required: this value type cannot be serialized.",v)
+ );
+ const getTypeIdById = (tid)=>{
+ switch(tid){
+ case TypeIds.number.id: return TypeIds.number;
+ case TypeIds.bigint.id: return TypeIds.bigint;
+ case TypeIds.boolean.id: return TypeIds.boolean;
+ case TypeIds.string.id: return TypeIds.string;
+ default: toss("Invalid type ID:",tid);
+ }
+ };
+ state.s11n.deserialize = function(){
+ ++metrics.s11n.deserialize.count;
+ const t = performance.now();
+ const argc = viewU8[0];
+ const rc = argc ? [] : null;
+ if(argc){
+ const typeIds = [];
+ let offset = 1, i, n, v;
+ for(i = 0; i < argc; ++i, ++offset){
+ typeIds.push(getTypeIdById(viewU8[offset]));
+ }
+ for(i = 0; i < argc; ++i){
+ const t = typeIds[i];
+ if(t.getter){
+ v = viewDV[t.getter](offset, state.littleEndian);
+ offset += t.size;
+ }else{/*String*/
+ n = viewDV.getInt32(offset, state.littleEndian);
+ offset += 4;
+ v = textDecoder.decode(viewU8.slice(offset, offset+n));
+ offset += n;
+ }
+ rc.push(v);
+ }
+ }
+ //log("deserialize:",argc, rc);
+ metrics.s11n.deserialize.time += performance.now() - t;
+ return rc;
+ };
+ state.s11n.serialize = function(...args){
+ const t = performance.now();
+ ++metrics.s11n.serialize.count;
+ if(args.length){
+ //log("serialize():",args);
+ const typeIds = [];
+ let i = 0, offset = 1;
+ viewU8[0] = args.length & 0xff /* header = # of args */;
+ for(; i < args.length; ++i, ++offset){
+ /* Write the TypeIds.id value into the next args.length
+ bytes. */
+ typeIds.push(getTypeId(args[i]));
+ viewU8[offset] = typeIds[i].id;
+ }
+ for(i = 0; i < args.length; ++i) {
+ /* Deserialize the following bytes based on their
+ corresponding TypeIds.id from the header. */
+ const t = typeIds[i];
+ if(t.setter){
+ viewDV[t.setter](offset, args[i], state.littleEndian);
+ offset += t.size;
+ }else{/*String*/
+ const s = textEncoder.encode(args[i]);
+ viewDV.setInt32(offset, s.byteLength, state.littleEndian);
+ offset += 4;
+ viewU8.set(s, offset);
+ offset += s.byteLength;
+ }
+ }
+ //log("serialize() result:",viewU8.slice(0,offset));
+ }else{
+ viewU8[0] = 0;
+ }
+ metrics.s11n.serialize.time += performance.now() - t;
+ };
+
+ state.s11n.storeException = state.asyncS11nExceptions
+ ? ((priority,e)=>{
+ if(priority<=state.asyncS11nExceptions){
+ state.s11n.serialize(e.message);
+ }
+ })
+ : ()=>{};
+
+ return state.s11n;
+}/*initS11n()*/;
+
+const waitLoop = async function f(){
+ const opHandlers = Object.create(null);
+ for(let k of Object.keys(state.opIds)){
+ const vi = vfsAsyncImpls[k];
+ if(!vi) continue;
+ const o = Object.create(null);
+ opHandlers[state.opIds[k]] = o;
+ o.key = k;
+ o.f = vi;
+ }
+ /**
+ waitTime is how long (ms) to wait for each Atomics.wait().
+ We need to wake up periodically to give the thread a chance
+ to do other things.
+ */
+ const waitTime = 1000;
+ while(!flagAsyncShutdown){
+ try {
+ if('timed-out'===Atomics.wait(
+ state.sabOPView, state.opIds.whichOp, 0, waitTime
+ )){
+ continue;
+ }
+ const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
+ Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
+ const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
+ const args = state.s11n.deserialize() || [];
+ state.s11n.serialize(/* clear s11n to keep the caller from
+ confusing this with an exception string
+ written by the upcoming operation */);
+ //warn("waitLoop() whichOp =",opId, hnd, args);
+ if(hnd.f) await hnd.f(...args);
+ else error("Missing callback for opId",opId);
+ }catch(e){
+ error('in waitLoop():',e);
+ }
+ }
+};
+
+navigator.storage.getDirectory().then(function(d){
+ const wMsg = (type)=>postMessage({type});
+ state.rootDir = d;
+ self.onmessage = function({data}){
+ switch(data.type){
+ case 'opfs-async-init':{
+ /* Receive shared state from synchronous partner */
+ const opt = data.args;
+ state.littleEndian = opt.littleEndian;
+ state.asyncS11nExceptions = opt.asyncS11nExceptions;
+ state.verbose = opt.verbose ?? 2;
+ state.fileBufferSize = opt.fileBufferSize;
+ state.sabS11nOffset = opt.sabS11nOffset;
+ state.sabS11nSize = opt.sabS11nSize;
+ state.sabOP = opt.sabOP;
+ state.sabOPView = new Int32Array(state.sabOP);
+ state.sabIO = opt.sabIO;
+ state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
+ state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
+ state.opIds = opt.opIds;
+ state.sq3Codes = opt.sq3Codes;
+ Object.keys(vfsAsyncImpls).forEach((k)=>{
+ if(!Number.isFinite(state.opIds[k])){
+ toss("Maintenance required: missing state.opIds[",k,"]");
+ }
+ });
+ initS11n();
+ metrics.reset();
+ log("init state",state);
+ wMsg('opfs-async-inited');
+ waitLoop();
+ break;
+ }
+ case 'opfs-async-restart':
+ if(flagAsyncShutdown){
+ warn("Restarting after opfs-async-shutdown. Might or might not work.");
+ flagAsyncShutdown = false;
+ waitLoop();
+ }
+ break;
+ case 'opfs-async-metrics':
+ metrics.dump();
+ break;
+ }
+ };
+ wMsg('opfs-async-loaded');
+}).catch((e)=>error("error initializing OPFS asyncer:",e));
diff --git a/ext/wasm/api/sqlite3-worker1-promiser.js b/ext/wasm/api/sqlite3-worker1-promiser.js
new file mode 100644
index 000000000..a77b0126d
--- /dev/null
+++ b/ext/wasm/api/sqlite3-worker1-promiser.js
@@ -0,0 +1,259 @@
+/*
+ 2022-08-24
+
+ The author disclaims copyright to this source code. In place of a
+ legal notice, here is a blessing:
+
+ * May you do good and not evil.
+ * May you find forgiveness for yourself and forgive others.
+ * May you share freely, never taking more than you give.
+
+ ***********************************************************************
+
+ This file implements a Promise-based proxy for the sqlite3 Worker
+ API #1. It is intended to be included either from the main thread or
+ a Worker, but only if (A) the environment supports nested Workers
+ and (B) it's _not_ a Worker which loads the sqlite3 WASM/JS
+ module. This file's features will load that module and provide a
+ slightly simpler client-side interface than the slightly-lower-level
+ Worker API does.
+
+ This script necessarily exposes one global symbol, but clients may
+ freely `delete` that symbol after calling it.
+*/
+'use strict';
+/**
+ Configures an sqlite3 Worker API #1 Worker such that it can be
+ manipulated via a Promise-based interface and returns a factory
+ function which returns Promises for communicating with the worker.
+ This proxy has an _almost_ identical interface to the normal
+ worker API, with any exceptions documented below.
+
+ It requires a configuration object with the following properties:
+
+ - `worker` (required): a Worker instance which loads
+ `sqlite3-worker1.js` or a functional equivalent. Note that the
+ promiser factory replaces the worker.onmessage property. This
+ config option may alternately be a function, in which case this
+ function re-assigns this property with the result of calling that
+ function, enabling delayed instantiation of a Worker.
+
+ - `onready` (optional, but...): this callback is called with no
+ arguments when the worker fires its initial
+ 'sqlite3-api'/'worker1-ready' message, which it does when
+ sqlite3.initWorker1API() completes its initialization. This is
+ the simplest way to tell the worker to kick off work at the
+ earliest opportunity.
+
+ - `onunhandled` (optional): a callback which gets passed the
+ message event object for any worker.onmessage() events which
+ are not handled by this proxy. Ideally that "should" never
+ happen, as this proxy aims to handle all known message types.
+
+ - `generateMessageId` (optional): a function which, when passed an
+ about-to-be-posted message object, generates a _unique_ message ID
+ for the message, which this API then assigns as the messageId
+ property of the message. It _must_ generate unique IDs on each call
+ so that dispatching can work. If not defined, a default generator
+ is used (which should be sufficient for most or all cases).
+
+ - `debug` (optional): a console.debug()-style function for logging
+ information about messages.
+
+ This function returns a stateful factory function with the
+ following interfaces:
+
+ - Promise function(messageType, messageArgs)
+ - Promise function({message object})
+
+ The first form expects the "type" and "args" values for a Worker
+ message. The second expects an object in the form {type:...,
+ args:...} plus any other properties the client cares to set. This
+ function will always set the `messageId` property on the object,
+ even if it's already set, and will set the `dbId` property to
+ `config.dbId` if it is _not_ set in the message object.
+
+ The function throws on error.
+
+ The function installs a temporary message listener, posts a
+ message to the configured Worker, and handles the message's
+ response via the temporary message listener. The then() callback
+ of the returned Promise is passed the `message.data` property from
+ the resulting message, i.e. the payload from the worker, stripped
+ of the lower-level event state which the onmessage() handler
+ receives.
+
+ Example usage:
+
+ ```
+ const config = {...};
+ const sq3Promiser = sqlite3Worker1Promiser(config);
+ sq3Promiser('open', {filename:"/foo.db"}).then(function(msg){
+ console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...}
+ });
+ sq3Promiser({type:'close'}).then((msg)=>{
+ console.log("close response",msg); // => {type:'close', result: {filename:'/foo.db'}, ...}
+ });
+ ```
+
+ Differences from Worker API #1:
+
+ - exec's {callback: STRING} option does not work via this
+ interface (it triggers an exception), but {callback: function}
+ does and works exactly like the STRING form does in the Worker:
+ the callback is called one time for each row of the result set,
+ passed the same worker message format as the worker API emits:
+
+ {type:typeString,
+ row:VALUE,
+ rowNumber:1-based-#,
+ columnNames: array}
+
+ Where `typeString` is an internally-synthesized message type string
+ used temporarily for worker message dispatching. It can be ignored
+ by all client code except that which tests this API. The `row`
+ property contains the row result in the form implied by the
+ `rowMode` option (defaulting to `'array'`). The `rowNumber` is a
+ 1-based integer value incremented by 1 on each call into th
+ callback.
+
+ At the end of the result set, the same event is fired with
+ (row=undefined, rowNumber=null) to indicate that
+ the end of the result set has been reached. Note that the rows
+ arrive via worker-posted messages, with all the implications
+ of that.
+*/
+self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){
+ // Inspired by: https://stackoverflow.com/a/52439530
+ if(1===arguments.length && 'function'===typeof arguments[0]){
+ const f = config;
+ config = Object.assign(Object.create(null), callee.defaultConfig);
+ config.onready = f;
+ }else{
+ config = Object.assign(Object.create(null), callee.defaultConfig, config);
+ }
+ const handlerMap = Object.create(null);
+ const noop = function(){};
+ const err = config.onerror
+ || noop /* config.onerror is intentionally undocumented
+ pending finding a less ambiguous name */;
+ const debug = config.debug || noop;
+ const idTypeMap = config.generateMessageId ? undefined : Object.create(null);
+ const genMsgId = config.generateMessageId || function(msg){
+ return msg.type+'#'+(idTypeMap[msg.type] = (idTypeMap[msg.type]||0) + 1);
+ };
+ const toss = (...args)=>{throw new Error(args.join(' '))};
+ if(!config.worker) config.worker = callee.defaultConfig.worker;
+ if('function'===typeof config.worker) config.worker = config.worker();
+ let dbId;
+ config.worker.onmessage = function(ev){
+ ev = ev.data;
+ debug('worker1.onmessage',ev);
+ let msgHandler = handlerMap[ev.messageId];
+ if(!msgHandler){
+ if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) {
+ /*fired one time when the Worker1 API initializes*/
+ if(config.onready) config.onready();
+ return;
+ }
+ msgHandler = handlerMap[ev.type] /* check for exec per-row callback */;
+ if(msgHandler && msgHandler.onrow){
+ msgHandler.onrow(ev);
+ return;
+ }
+ if(config.onunhandled) config.onunhandled(arguments[0]);
+ else err("sqlite3Worker1Promiser() unhandled worker message:",ev);
+ return;
+ }
+ delete handlerMap[ev.messageId];
+ switch(ev.type){
+ case 'error':
+ msgHandler.reject(ev);
+ return;
+ case 'open':
+ if(!dbId) dbId = ev.dbId;
+ break;
+ case 'close':
+ if(ev.dbId===dbId) dbId = undefined;
+ break;
+ default:
+ break;
+ }
+ try {msgHandler.resolve(ev)}
+ catch(e){msgHandler.reject(e)}
+ }/*worker.onmessage()*/;
+ return function(/*(msgType, msgArgs) || (msgEnvelope)*/){
+ let msg;
+ if(1===arguments.length){
+ msg = arguments[0];
+ }else if(2===arguments.length){
+ msg = {
+ type: arguments[0],
+ args: arguments[1]
+ };
+ }else{
+ toss("Invalid arugments for sqlite3Worker1Promiser()-created factory.");
+ }
+ if(!msg.dbId) msg.dbId = dbId;
+ msg.messageId = genMsgId(msg);
+ msg.departureTime = performance.now();
+ const proxy = Object.create(null);
+ proxy.message = msg;
+ let rowCallbackId /* message handler ID for exec on-row callback proxy */;
+ if('exec'===msg.type && msg.args){
+ if('function'===typeof msg.args.callback){
+ rowCallbackId = msg.messageId+':row';
+ proxy.onrow = msg.args.callback;
+ msg.args.callback = rowCallbackId;
+ handlerMap[rowCallbackId] = proxy;
+ }else if('string' === typeof msg.args.callback){
+ toss("exec callback may not be a string when using the Promise interface.");
+ /**
+ Design note: the reason for this limitation is that this
+ API takes over worker.onmessage() and the client has no way
+ of adding their own message-type handlers to it. Per-row
+ callbacks are implemented as short-lived message.type
+ mappings for worker.onmessage().
+
+ We "could" work around this by providing a new
+ config.fallbackMessageHandler (or some such) which contains
+ a map of event type names to callbacks. Seems like overkill
+ for now, seeing as the client can pass callback functions
+ to this interface (whereas the string-form "callback" is
+ needed for the over-the-Worker interface).
+ */
+ }
+ }
+ //debug("requestWork", msg);
+ let p = new Promise(function(resolve, reject){
+ proxy.resolve = resolve;
+ proxy.reject = reject;
+ handlerMap[msg.messageId] = proxy;
+ debug("Posting",msg.type,"message to Worker dbId="+(dbId||'default')+':',msg);
+ config.worker.postMessage(msg);
+ });
+ if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]);
+ return p;
+ };
+}/*sqlite3Worker1Promiser()*/;
+self.sqlite3Worker1Promiser.defaultConfig = {
+ worker: function(){
+ let theJs = "sqlite3-worker1.js";
+ if(this.currentScript){
+ const src = this.currentScript.src.split('/');
+ src.pop();
+ theJs = src.join('/')+'/' + theJs;
+ //console.warn("promiser currentScript, theJs =",this.currentScript,theJs);
+ }else{
+ //console.warn("promiser self.location =",self.location);
+ const urlParams = new URL(self.location.href).searchParams;
+ if(urlParams.has('sqlite3.dir')){
+ theJs = urlParams.get('sqlite3.dir') + '/' + theJs;
+ }
+ }
+ return new Worker(theJs + self.location.search);
+ }.bind({
+ currentScript: self?.document?.currentScript
+ }),
+ onerror: (...args)=>console.error('worker1 promiser error',...args)
+};
diff --git a/ext/wasm/api/sqlite3-worker1.js b/ext/wasm/api/sqlite3-worker1.js
new file mode 100644
index 000000000..bc860300b
--- /dev/null
+++ b/ext/wasm/api/sqlite3-worker1.js
@@ -0,0 +1,53 @@
+/*
+ 2022-05-23
+
+ The author disclaims copyright to this source code. In place of a
+ legal notice, here is a blessing:
+
+ * May you do good and not evil.
+ * May you find forgiveness for yourself and forgive others.
+ * May you share freely, never taking more than you give.
+
+ ***********************************************************************
+
+ This is a JS Worker file for the main sqlite3 api. It loads
+ sqlite3.js, initializes the module, and postMessage()'s a message
+ after the module is initialized:
+
+ {type: 'sqlite3-api', result: 'worker1-ready'}
+
+ This seemingly superfluous level of indirection is necessary when
+ loading sqlite3.js via a Worker. Instantiating a worker with new
+ Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to
+ initialize the module due to a timing/order-of-operations conflict
+ (and that symbol is not exported in a way that a Worker loading it
+ that way can see it). Thus JS code wanting to load the sqlite3
+ Worker-specific API needs to pass _this_ file (or equivalent) to the
+ Worker constructor and then listen for an event in the form shown
+ above in order to know when the module has completed initialization.
+
+ This file accepts a couple of URL arguments to adjust how it loads
+ sqlite3.js:
+
+ - `sqlite3.js`, if set, is used as the URI to `sqlite3.js` and it
+ may contain path elements, e.g. `sqlite3.js=foo/bar/my-sqlite3.js`.
+ - `sqlite3.dir`, if set, treats the given directory name as the
+ directory from which `sqlite3.js` will be loaded.
+
+ By default is loads 'sqlite3.js'.
+*/
+"use strict";
+(()=>{
+ const urlParams = new URL(self.location.href).searchParams;
+ let theJs = 'sqlite3.js';
+ if(urlParams.has('sqlite3.js')){
+ theJs = urlParams.get('sqlite3.js');
+ }else if(urlParams.has('sqlite3.dir')){
+ theJs = urlParams.get('sqlite3.dir') + '/' + theJs;
+ }
+ importScripts(theJs);
+ sqlite3InitModule().then((sqlite3)=>{
+ sqlite3.capi.sqlite3_wasmfs_opfs_dir();
+ sqlite3.initWorker1API();
+ });
+})();