diff options
Diffstat (limited to 'ext/wasm/api')
-rw-r--r-- | ext/wasm/api/extern-post-js.js | 45 | ||||
-rw-r--r-- | ext/wasm/api/pre-js.js | 44 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-opfs.js | 14 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-prologue.js | 16 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-opfs-async-proxy.js | 730 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-worker1-promiser.js | 259 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-worker1.js | 53 |
7 files changed, 1149 insertions, 12 deletions
diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js index 7dba03b3a..5f5f72d4b 100644 --- a/ext/wasm/api/extern-post-js.js +++ b/ext/wasm/api/extern-post-js.js @@ -16,6 +16,27 @@ if(!originalInit){ throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); } + /** + We need to add some state which our custom Module.locateFile() + can see, but an Emscripten limitation currently prevents us from + attaching it to the sqlite3InitModule function object: + + https://github.com/emscripten-core/emscripten/issues/18071 + + The only current workaround is to temporarily stash this state + into the global scope and delete it when sqlite3InitModule() + is called. + */ + const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ + moduleScript: self?.document?.currentScript, + isWorker: (!self.document && self.window !== self), + location: self.location, + urlParams: new URL(self.location.href).searchParams + }); + if(initModuleState.urlParams.has('sqlite3.dir')){ + initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/'; + }; + self.sqlite3InitModule = (...args)=>{ //console.warn("Using replaced sqlite3InitModule()",self.location); return originalInit(...args).then((EmscriptenModule)=>{ @@ -32,6 +53,8 @@ Emscripten details. */ return EmscriptenModule; } + EmscriptenModule.sqlite3.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); const f = EmscriptenModule.sqlite3.asyncPostInit; delete EmscriptenModule.sqlite3.asyncPostInit; return f(); @@ -41,13 +64,19 @@ }); }; self.sqlite3InitModule.ready = originalInit.ready; - //console.warn("Replaced sqlite3InitModule()"); -})(); -if(0){ - console.warn("self.location.href =",self.location.href); - if('undefined' !== typeof document){ - console.warn("document.currentScript.src =", - document?.currentScript?.src); + if(self.sqlite3InitModuleState.moduleScript){ + const sim = self.sqlite3InitModuleState; + let src = sim.moduleScript.src.split('/'); + src.pop(); + sim.scriptDir = src.join('/') + '/'; } -} + if(0){ + console.warn("Replaced sqlite3InitModule()"); + console.warn("self.location.href =",self.location.href); + if('undefined' !== typeof document){ + console.warn("document.currentScript.src =", + document?.currentScript?.src); + } + } +})(); diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js index b6630416d..c07d0373c 100644 --- a/ext/wasm/api/pre-js.js +++ b/ext/wasm/api/pre-js.js @@ -4,9 +4,49 @@ This file is intended to be prepended to the sqlite3.js build using Emscripten's --pre-js=THIS_FILE flag (or equivalent). */ + +// See notes in extern-post-js.js +const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null); +delete self.sqlite3InitModuleState; + +/** + This custom locateFile() tries to figure out where to load `path` + from. The intent is to provide a way for foo/bar/X.js loaded from a + Worker constructor or importScripts() to be able to resolve + foo/bar/X.wasm (in the latter case, with some help): + + 1) If URL param named the same as `path` is set, it is returned. + + 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path) + is returned (note that it's assumed to end with '/'). + + 3) If this code is running in the main UI thread AND it was loaded + from a SCRIPT tag, the directory part of that URL is used + as the prefix. (This form of resolution unfortunately does not + function for scripts loaded via importScripts().) + + 4) If none of the above apply, (prefix+path) is returned. +*/ Module['locateFile'] = function(path, prefix) { - return prefix + path; -}; + let theFile; + const up = this.urlParams; + if(0){ + console.warn("locateFile(",arguments[0], ',', arguments[1],")", + 'self.location =',self.location, + 'sqlite3InitModuleState.scriptDir =',this.scriptDir, + 'up.entries() =',Array.from(up.entries())); + } + if(up.has(path)){ + theFile = up.get(path); + }else if(this.sqlite3Dir){ + theFile = this.sqlite3Dir + path; + }else if(this.scriptDir){ + theFile = this.scriptDir + path; + }else{ + theFile = prefix + path; + } + return theFile; +}.bind(sqlite3InitModuleState); /** Bug warning: this xInstantiateWasm bit must remain disabled diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 98defe4fb..9f5302454 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -98,6 +98,9 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ options.proxyUri = callee.defaultProxyUri; } + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } const thePromise = new Promise(function(promiseResolve, promiseReject_){ const loggers = { 0:console.error.bind(console), @@ -1092,9 +1095,18 @@ installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; //console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri); self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ + return; + } try{ + let proxyJs = installOpfsVfs.defaultProxyUri; + if(sqlite3.scriptInfo.sqlite3Dir){ + installOpfsVfs.defaultProxyUri = + sqlite3.scriptInfo.sqlite3Dir + proxyJs; + //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + } return installOpfsVfs().catch((e)=>{ - console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e); + console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); }); }catch(e){ console.error("installOpfsVfs() exception:",e); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index f8c0f024e..3980e0ad4 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1334,7 +1334,20 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( //while(lip.length) p = p.then(lip.shift()); //return p.then(()=>sqlite3); return Promise.all(lip).then(()=>sqlite3); - } + }, + /** + scriptInfo ideally gets injected into this object by the + infrastructure which assembles the JS/WASM module. It contains + state which must be collected before sqlite3ApiBootstrap() can + be declared. It is not necessarily available to any + sqlite3ApiBootstrap.initializers but "should" be in place (if + it's added at all) by the time that + sqlite3ApiBootstrap.initializersAsync is processed. + + This state is not part of the public API, only intended for use + with the sqlite3 API bootstrapping and wasm-loading process. + */ + scriptInfo: undefined }; try{ sqlite3ApiBootstrap.initializers.forEach((f)=>{ @@ -1410,3 +1423,4 @@ self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); value which will be stored here. */ self.sqlite3ApiBootstrap.sqlite3 = undefined; + diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js new file mode 100644 index 000000000..5b29aa3ea --- /dev/null +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -0,0 +1,730 @@ +/* + 2022-09-16 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + An Worker which manages asynchronous OPFS handles on behalf of a + synchronous API which controls it via a combination of Worker + messages, SharedArrayBuffer, and Atomics. It is the asynchronous + counterpart of the API defined in sqlite3-api-opfs.js. + + Highly indebted to: + + https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js + + for demonstrating how to use the OPFS APIs. + + This file is to be loaded as a Worker. It does not have any direct + access to the sqlite3 JS/WASM bits, so any bits which it needs (most + notably SQLITE_xxx integer codes) have to be imported into it via an + initialization process. +*/ +'use strict'; +const toss = function(...args){throw new Error(args.join(' '))}; +if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); +} + +/** + Will hold state copied to this object from the syncronous side of + this API. +*/ +const state = Object.create(null); +/** + verbose: + + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors +*/ +state.verbose = 2; + +const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) +}; +const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); +}; +const log = (...args)=>logImpl(2, ...args); +const warn = (...args)=>logImpl(1, ...args); +const error = (...args)=>logImpl(0, ...args); +const metrics = Object.create(null); +metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; +}; +metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); +}; + +/** + Map of sqlite3_file pointers (integers) to metadata related to a + given OPFS file handles. The pointers are, in this side of the + interface, opaque file handle IDs provided by the synchronous + part of this constellation. Each value is an object with a structure + demonstrated in the xOpen() impl. +*/ +const __openFiles = Object.create(null); + +/** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. +*/ +const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; +}; + +/** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. +*/ +const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; +}; + +/** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object), lazily opening + it if needed. + + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. +*/ +const getSyncHandle = async (fh)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 3; + let i = 1, ms = 300; + for(; true; ms *= ++i){ + try { + //if(1===i) toss("Just testing."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + toss("Error getting sync handle.",maxTries, + "attempts failed. ",fh.filenameAbs, ":", e.message); + throw e; + } + warn("Error getting sync handle. Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + } + } + log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + } + return fh.syncHandle; +}; + +/** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. +*/ +const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + return h.close(); + } +}; + +/** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. +*/ +const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",state.opIds.rc,",",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); +}; + +/** + Throws if fh is a file-holding object which is flagged as read-only. +*/ +const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); +}; + +/** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. +*/ +const __mTimer = Object.create(null); +__mTimer.op = undefined; +__mTimer.start = undefined; +const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; +}; +const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start +); +const __wTimer = Object.create(null); +__wTimer.op = undefined; +__wTimer.start = undefined; +const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); +}; +const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start +); + +/** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. +*/ +let flagAsyncShutdown = false; + + +/** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. +*/ +const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); + try { + await getDirForFilename(dirname+"/filepart", true); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries sqlite3 + calls for. We can essentially just tell if the file is + accessible, but if it is it's automatically writable (unless + it's locked, which we cannot(?) know without trying to open + it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid){ + const opName = 'xClose'; + mTimeStart(opName); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart('xClose'); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; + } + wTimeEnd(); + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let sz; + wTimeStart('xFileSize'); + try{ + sz = await (await getSyncHandle(fh)).getSize(); + state.s11n.serialize(Number(sz)); + sz = 0; + }catch(e){ + state.s11n.storeException(2,e); + sz = state.sq3Codes.SQLITE_IOERR; + } + wTimeEnd(); + storeAndNotify('xFileSize', sz); + mTimeEnd(); + }, + xLock: async function(fid,lockType){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { await getSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){ + const opName = 'xOpen'; + mTimeStart(opName); + const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + /** + wa-sqlite, at this point, grabs a SyncAccessHandle and + assigns it to the syncHandle prop of the file state + object, but only for certain cases and it's unclear why it + places that limitation on it. + */ + wTimeEnd(); + __openFiles[fid] = Object.assign(Object.create(null),{ + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: deleteOnClose + }); + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); + } + mTimeEnd(); + }, + xRead: async function(fid,n,offset){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh)).read( + fh.sabView.subarray(0, n), + {at: Number(offset)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_READ; + } + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh)).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; + } + wTimeEnd(); + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid,lockType){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR; + } + wTimeEnd(); + } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid,n,offset){ + mTimeStart('xWrite'); + let rc; + wTimeStart('xWrite'); + try{ + const fh = __openFiles[fid]; + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh)) + .write(fh.sabView.subarray(0, n), + {at: Number(offset)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_WRITE; + } + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); + } +}/*vfsAsyncImpls*/; + +const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + state.s11n.deserialize = function(){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize(e.message); + } + }) + : ()=>{}; + + return state.s11n; +}/*initS11n()*/; + +const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wake up periodically to give the thread a chance + to do other things. + */ + const waitTime = 1000; + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize() || []; + state.s11n.serialize(/* clear s11n to keep the caller from + confusing this with an exception string + written by the upcoming operation */); + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } + } +}; + +navigator.storage.getDirectory().then(function(d){ + const wMsg = (type)=>postMessage({type}); + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + state.littleEndian = opt.littleEndian; + state.asyncS11nExceptions = opt.asyncS11nExceptions; + state.verbose = opt.verbose ?? 2; + state.fileBufferSize = opt.fileBufferSize; + state.sabS11nOffset = opt.sabS11nOffset; + state.sabS11nSize = opt.sabS11nSize; + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wMsg('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wMsg('opfs-async-loaded'); +}).catch((e)=>error("error initializing OPFS asyncer:",e)); diff --git a/ext/wasm/api/sqlite3-worker1-promiser.js b/ext/wasm/api/sqlite3-worker1-promiser.js new file mode 100644 index 000000000..a77b0126d --- /dev/null +++ b/ext/wasm/api/sqlite3-worker1-promiser.js @@ -0,0 +1,259 @@ +/* + 2022-08-24 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Promise-based proxy for the sqlite3 Worker + API #1. It is intended to be included either from the main thread or + a Worker, but only if (A) the environment supports nested Workers + and (B) it's _not_ a Worker which loads the sqlite3 WASM/JS + module. This file's features will load that module and provide a + slightly simpler client-side interface than the slightly-lower-level + Worker API does. + + This script necessarily exposes one global symbol, but clients may + freely `delete` that symbol after calling it. +*/ +'use strict'; +/** + Configures an sqlite3 Worker API #1 Worker such that it can be + manipulated via a Promise-based interface and returns a factory + function which returns Promises for communicating with the worker. + This proxy has an _almost_ identical interface to the normal + worker API, with any exceptions documented below. + + It requires a configuration object with the following properties: + + - `worker` (required): a Worker instance which loads + `sqlite3-worker1.js` or a functional equivalent. Note that the + promiser factory replaces the worker.onmessage property. This + config option may alternately be a function, in which case this + function re-assigns this property with the result of calling that + function, enabling delayed instantiation of a Worker. + + - `onready` (optional, but...): this callback is called with no + arguments when the worker fires its initial + 'sqlite3-api'/'worker1-ready' message, which it does when + sqlite3.initWorker1API() completes its initialization. This is + the simplest way to tell the worker to kick off work at the + earliest opportunity. + + - `onunhandled` (optional): a callback which gets passed the + message event object for any worker.onmessage() events which + are not handled by this proxy. Ideally that "should" never + happen, as this proxy aims to handle all known message types. + + - `generateMessageId` (optional): a function which, when passed an + about-to-be-posted message object, generates a _unique_ message ID + for the message, which this API then assigns as the messageId + property of the message. It _must_ generate unique IDs on each call + so that dispatching can work. If not defined, a default generator + is used (which should be sufficient for most or all cases). + + - `debug` (optional): a console.debug()-style function for logging + information about messages. + + This function returns a stateful factory function with the + following interfaces: + + - Promise function(messageType, messageArgs) + - Promise function({message object}) + + The first form expects the "type" and "args" values for a Worker + message. The second expects an object in the form {type:..., + args:...} plus any other properties the client cares to set. This + function will always set the `messageId` property on the object, + even if it's already set, and will set the `dbId` property to + `config.dbId` if it is _not_ set in the message object. + + The function throws on error. + + The function installs a temporary message listener, posts a + message to the configured Worker, and handles the message's + response via the temporary message listener. The then() callback + of the returned Promise is passed the `message.data` property from + the resulting message, i.e. the payload from the worker, stripped + of the lower-level event state which the onmessage() handler + receives. + + Example usage: + + ``` + const config = {...}; + const sq3Promiser = sqlite3Worker1Promiser(config); + sq3Promiser('open', {filename:"/foo.db"}).then(function(msg){ + console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...} + }); + sq3Promiser({type:'close'}).then((msg)=>{ + console.log("close response",msg); // => {type:'close', result: {filename:'/foo.db'}, ...} + }); + ``` + + Differences from Worker API #1: + + - exec's {callback: STRING} option does not work via this + interface (it triggers an exception), but {callback: function} + does and works exactly like the STRING form does in the Worker: + the callback is called one time for each row of the result set, + passed the same worker message format as the worker API emits: + + {type:typeString, + row:VALUE, + rowNumber:1-based-#, + columnNames: array} + + Where `typeString` is an internally-synthesized message type string + used temporarily for worker message dispatching. It can be ignored + by all client code except that which tests this API. The `row` + property contains the row result in the form implied by the + `rowMode` option (defaulting to `'array'`). The `rowNumber` is a + 1-based integer value incremented by 1 on each call into th + callback. + + At the end of the result set, the same event is fired with + (row=undefined, rowNumber=null) to indicate that + the end of the result set has been reached. Note that the rows + arrive via worker-posted messages, with all the implications + of that. +*/ +self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ + // Inspired by: https://stackoverflow.com/a/52439530 + if(1===arguments.length && 'function'===typeof arguments[0]){ + const f = config; + config = Object.assign(Object.create(null), callee.defaultConfig); + config.onready = f; + }else{ + config = Object.assign(Object.create(null), callee.defaultConfig, config); + } + const handlerMap = Object.create(null); + const noop = function(){}; + const err = config.onerror + || noop /* config.onerror is intentionally undocumented + pending finding a less ambiguous name */; + const debug = config.debug || noop; + const idTypeMap = config.generateMessageId ? undefined : Object.create(null); + const genMsgId = config.generateMessageId || function(msg){ + return msg.type+'#'+(idTypeMap[msg.type] = (idTypeMap[msg.type]||0) + 1); + }; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if(!config.worker) config.worker = callee.defaultConfig.worker; + if('function'===typeof config.worker) config.worker = config.worker(); + let dbId; + config.worker.onmessage = function(ev){ + ev = ev.data; + debug('worker1.onmessage',ev); + let msgHandler = handlerMap[ev.messageId]; + if(!msgHandler){ + if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { + /*fired one time when the Worker1 API initializes*/ + if(config.onready) config.onready(); + return; + } + msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; + if(msgHandler && msgHandler.onrow){ + msgHandler.onrow(ev); + return; + } + if(config.onunhandled) config.onunhandled(arguments[0]); + else err("sqlite3Worker1Promiser() unhandled worker message:",ev); + return; + } + delete handlerMap[ev.messageId]; + switch(ev.type){ + case 'error': + msgHandler.reject(ev); + return; + case 'open': + if(!dbId) dbId = ev.dbId; + break; + case 'close': + if(ev.dbId===dbId) dbId = undefined; + break; + default: + break; + } + try {msgHandler.resolve(ev)} + catch(e){msgHandler.reject(e)} + }/*worker.onmessage()*/; + return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + let msg; + if(1===arguments.length){ + msg = arguments[0]; + }else if(2===arguments.length){ + msg = { + type: arguments[0], + args: arguments[1] + }; + }else{ + toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); + } + if(!msg.dbId) msg.dbId = dbId; + msg.messageId = genMsgId(msg); + msg.departureTime = performance.now(); + const proxy = Object.create(null); + proxy.message = msg; + let rowCallbackId /* message handler ID for exec on-row callback proxy */; + if('exec'===msg.type && msg.args){ + if('function'===typeof msg.args.callback){ + rowCallbackId = msg.messageId+':row'; + proxy.onrow = msg.args.callback; + msg.args.callback = rowCallbackId; + handlerMap[rowCallbackId] = proxy; + }else if('string' === typeof msg.args.callback){ + toss("exec callback may not be a string when using the Promise interface."); + /** + Design note: the reason for this limitation is that this + API takes over worker.onmessage() and the client has no way + of adding their own message-type handlers to it. Per-row + callbacks are implemented as short-lived message.type + mappings for worker.onmessage(). + + We "could" work around this by providing a new + config.fallbackMessageHandler (or some such) which contains + a map of event type names to callbacks. Seems like overkill + for now, seeing as the client can pass callback functions + to this interface (whereas the string-form "callback" is + needed for the over-the-Worker interface). + */ + } + } + //debug("requestWork", msg); + let p = new Promise(function(resolve, reject){ + proxy.resolve = resolve; + proxy.reject = reject; + handlerMap[msg.messageId] = proxy; + debug("Posting",msg.type,"message to Worker dbId="+(dbId||'default')+':',msg); + config.worker.postMessage(msg); + }); + if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]); + return p; + }; +}/*sqlite3Worker1Promiser()*/; +self.sqlite3Worker1Promiser.defaultConfig = { + worker: function(){ + let theJs = "sqlite3-worker1.js"; + if(this.currentScript){ + const src = this.currentScript.src.split('/'); + src.pop(); + theJs = src.join('/')+'/' + theJs; + //console.warn("promiser currentScript, theJs =",this.currentScript,theJs); + }else{ + //console.warn("promiser self.location =",self.location); + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + } + return new Worker(theJs + self.location.search); + }.bind({ + currentScript: self?.document?.currentScript + }), + onerror: (...args)=>console.error('worker1 promiser error',...args) +}; diff --git a/ext/wasm/api/sqlite3-worker1.js b/ext/wasm/api/sqlite3-worker1.js new file mode 100644 index 000000000..bc860300b --- /dev/null +++ b/ext/wasm/api/sqlite3-worker1.js @@ -0,0 +1,53 @@ +/* + 2022-05-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This is a JS Worker file for the main sqlite3 api. It loads + sqlite3.js, initializes the module, and postMessage()'s a message + after the module is initialized: + + {type: 'sqlite3-api', result: 'worker1-ready'} + + This seemingly superfluous level of indirection is necessary when + loading sqlite3.js via a Worker. Instantiating a worker with new + Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to + initialize the module due to a timing/order-of-operations conflict + (and that symbol is not exported in a way that a Worker loading it + that way can see it). Thus JS code wanting to load the sqlite3 + Worker-specific API needs to pass _this_ file (or equivalent) to the + Worker constructor and then listen for an event in the form shown + above in order to know when the module has completed initialization. + + This file accepts a couple of URL arguments to adjust how it loads + sqlite3.js: + + - `sqlite3.js`, if set, is used as the URI to `sqlite3.js` and it + may contain path elements, e.g. `sqlite3.js=foo/bar/my-sqlite3.js`. + - `sqlite3.dir`, if set, treats the given directory name as the + directory from which `sqlite3.js` will be loaded. + + By default is loads 'sqlite3.js'. +*/ +"use strict"; +(()=>{ + const urlParams = new URL(self.location.href).searchParams; + let theJs = 'sqlite3.js'; + if(urlParams.has('sqlite3.js')){ + theJs = urlParams.get('sqlite3.js'); + }else if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + importScripts(theJs); + sqlite3InitModule().then((sqlite3)=>{ + sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + sqlite3.initWorker1API(); + }); +})(); |