diff options
Diffstat (limited to 'ext/wasm/scratchpad-opfs-worker2.js')
| -rw-r--r-- | ext/wasm/scratchpad-opfs-worker2.js | 494 |
1 files changed, 494 insertions, 0 deletions
diff --git a/ext/wasm/scratchpad-opfs-worker2.js b/ext/wasm/scratchpad-opfs-worker2.js new file mode 100644 index 000000000..47ace63de --- /dev/null +++ b/ext/wasm/scratchpad-opfs-worker2.js @@ -0,0 +1,494 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + An experiment for wasmfs/opfs. This file MUST be in the same dir as + the sqlite3.js emscripten module or that module won't be able to + resolve the relative URIs (importScript()'s relative URI handling + is, quite frankly, broken). +*/ +'use strict'; + +const toss = function(...args){throw new Error(args.join(' '))}; +/** + Posts a message in the form {type,data} unless passed more than 2 + args, in which case it posts {type, data:[arg1...argN]}. +*/ +const wMsg = function(type,data){ + postMessage({ + type, + data: arguments.length<3 + ? data + : Array.prototype.slice.call(arguments,1) + }); +}; + +const stdout = function(...args){ + wMsg('stdout',args); + console.log(...args); +}; +const stderr = function(...args){ + wMsg('stderr',args); + console.error(...args); +}; + +const log = console.log.bind(console); +const warn = console.warn.bind(console); +const error = console.error.bind(console); + + +const initOpfsBits = async function(sqlite3){ + if(!self.importScripts || !self.FileSystemFileHandle){ + //|| !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ + // ^^^ sync API is not required with WASMFS/OPFS backend. + warn("OPFS is not available in this environment."); + return; + }else if(!sqlite3.capi.wasm.bigIntEnabled){ + error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); + return; + } + //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); + //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); + const capi = sqlite3.capi, + wasm = capi.wasm; + const sqlite3_vfs = capi.sqlite3_vfs + || toss("Missing sqlite3.capi.sqlite3_vfs object."); + const sqlite3_file = capi.sqlite3_file + || toss("Missing sqlite3.capi.sqlite3_file object."); + const sqlite3_io_methods = capi.sqlite3_io_methods + || toss("Missing sqlite3.capi.sqlite3_io_methods object."); + const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); + const debug = console.debug.bind(console), + log = console.log.bind(console); + warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const oVfs = new sqlite3_vfs(); + const oIom = new sqlite3_io_methods(); + oVfs.$iVersion = 2/*yes, two*/; + oVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + oVfs.$mxPathname = 1024/*sure, why not?*/; + oVfs.$zName = wasm.allocCString("opfs"); + oVfs.ondispose = [ + '$zName', oVfs.$zName, + 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) + ]; + if(dVfs){ + oVfs.$xSleep = dVfs.$xSleep; + oVfs.$xRandomness = dVfs.$xRandomness; + } + // All C-side memory of oVfs is zeroed out, but just to be explicit: + oVfs.$xDlOpen = oVfs.$xDlError = oVfs.$xDlSym = oVfs.$xDlClose = null; + + /** + Pedantic sidebar about oVfs.ondispose: the entries in that array + are items to clean up when oVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the oVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + const fProxy = 1 + // We can remove this proxy middle-man once the VFS is working + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + /** + Map of sqlite3_file pointers to OPFS handles. + */ + const __opfsHandles = Object.create(null); + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + const rootDir = await navigator.storage.getDirectory(); + log("rootDir =",rootDir); + + //////////////////////////////////////////////////////////////////////// + // Set up OPFS VFS methods... + let inst = installMethod(oVfs); + inst('xOpen', function(pVfs, zName, pFile, flags, pOutFlags){ + const f = new sqlite3_file(pFile); + f.$pMethods = oIom.pointer; + __opfsHandles[pFile] = f; + f.opfsHandle = null /* TODO */; + if(flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + f.deleteOnClose = true; + } + f.filename = zName ? wasm.cstringToJs(zName) : 'sqlite3-xOpen-'+randomFilename(); + error("OPFS sqlite3_vfs::xOpen is not yet full implemented."); + return capi.SQLITE_IOERR; + }) + ('xFullPathname', function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i<nOut ? 0 : capi.SQLITE_CANTOPEN + /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/; + }) + ('xAccess', function(pVfs,zName,flags,pOut){ + error("OPFS sqlite3_vfs::xAccess is not yet implemented."); + let fileExists = 0; + switch(flags){ + case capi.SQLITE_ACCESS_EXISTS: break; + case capi.SQLITE_ACCESS_READWRITE: break; + case capi.SQLITE_ACCESS_READ/*docs say this is never used*/: + default: + error("Unexpected flags value for sqlite3_vfs::xAccess():",flags); + return capi.SQLITE_MISUSE; + } + wasm.setMemValue(pOut, fileExists, 'i32'); + return 0; + }) + ('xDelete', function(pVfs, zName, doSyncDir){ + error("OPFS sqlite3_vfs::xDelete is not yet implemented."); + // https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/modules/file_system_access/file_system_handle.idl + // ==> remove() + return capi.SQLITE_IOERR; + }) + ('xGetLastError', function(pVfs,nOut,pOut){ + debug("OPFS sqlite3_vfs::xGetLastError() has nothing sensible to return."); + return 0; + }) + ('xCurrentTime', function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }) + ('xCurrentTimeInt64',function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }); + if(!oVfs.$xSleep){ + inst('xSleep', function(pVfs,ms){ + error("sqlite3_vfs::xSleep(",ms,") cannot be implemented from "+ + "JS and we have no default VFS to copy the impl from."); + return 0; + }); + } + if(!oVfs.$xRandomness){ + inst('xRandomness', function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }); + } + + //////////////////////////////////////////////////////////////////////// + // Set up OPFS sqlite3_io_methods... + inst = installMethod(oIom); + inst('xClose', async function(pFile){ + warn("xClose(",arguments,") uses await"); + const f = __opfsHandles[pFile]; + delete __opfsHandles[pFile]; + if(f.opfsHandle){ + await f.opfsHandle.close(); + if(f.deleteOnClose){ + // TODO + } + } + f.dispose(); + return 0; + }) + ('xRead', /*i(ppij)*/function(pFile,pDest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + try { + const f = __opfsHandles[pFile]; + const heap = wasm.heap8u(); + const b = new Uint8Array(heap.buffer, pDest, n); + const nRead = f.opfsHandle.read(b, {at: offset}); + if(nRead<n){ + // MUST zero-fill short reads (per the docs) + heap.fill(0, dest + nRead, n - nRead); + } + return 0; + }catch(e){ + error("xRead(",arguments,") failed:",e); + return capi.SQLITE_IOERR_READ; + } + }) + ('xWrite', /*i(ppij)*/function(pFile,pSrc,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + try { + const f = __opfsHandles[pFile]; + const b = new Uint8Array(wasm.heap8u().buffer, pSrc, n); + const nOut = f.opfsHandle.write(b, {at: offset}); + if(nOut<n){ + error("xWrite(",arguments,") short write!"); + return capi.SQLITE_IOERR_WRITE; + } + return 0; + }catch(e){ + error("xWrite(",arguments,") failed:",e); + return capi.SQLITE_IOERR_WRITE; + } + }) + ('xTruncate', /*i(pj)*/async function(pFile,sz){ + /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ + try{ + warn("xTruncate(",arguments,") uses await"); + const f = __opfsHandles[pFile]; + await f.opfsHandle.truncate(sz); + return 0; + } + catch(e){ + error("xTruncate(",arguments,") failed:",e); + return capi.SQLITE_IOERR_TRUNCATE; + } + }) + ('xSync', /*i(pi)*/async function(pFile,flags){ + /* int (*xSync)(sqlite3_file*, int flags) */ + try { + warn("xSync(",arguments,") uses await"); + const f = __opfsHandles[pFile]; + await f.opfsHandle.flush(); + return 0; + }catch(e){ + error("xSync(",arguments,") failed:",e); + return capi.SQLITE_IOERR_SYNC; + } + }) + ('xFileSize', /*i(pp)*/async function(pFile,pSz){ + /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ + try { + warn("xFileSize(",arguments,") uses await"); + const f = __opfsHandles[pFile]; + const fsz = await f.opfsHandle.getSize(); + capi.wasm.setMemValue(pSz, fsz,'i64'); + return 0; + }catch(e){ + error("xFileSize(",arguments,") failed:",e); + return capi.SQLITE_IOERR_SEEK; + } + }) + ('xLock', /*i(pi)*/function(pFile,lockType){ + /* int (*xLock)(sqlite3_file*, int) */ + // Opening a handle locks it automatically. + warn("xLock(",arguments,") is a no-op"); + return 0; + }) + ('xUnlock', /*i(pi)*/function(pFile,lockType){ + /* int (*xUnlock)(sqlite3_file*, int) */ + // Opening a handle locks it automatically. + warn("xUnlock(",arguments,") is a no-op"); + return 0; + }) + ('xCheckReservedLock', /*i(pp)*/function(pFile,pOut){ + /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ + // Exclusive lock is automatically acquired when opened + warn("xCheckReservedLock(",arguments,") is a no-op"); + wasm.setMemValue(pOut,1,'i32'); + return 0; + }) + ('xFileControl', /*i(pip)*/function(pFile,op,pArg){ + /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ + debug("xFileControl(",arguments,") is a no-op"); + return capi.SQLITE_NOTFOUND; + }) + ('xDeviceCharacteristics',/*i(p)*/function(pFile){ + /* int (*xDeviceCharacteristics)(sqlite3_file*) */ + debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }); + // xSectorSize may be NULL + //('xSectorSize', function(pFile){ + // /* int (*xSectorSize)(sqlite3_file*) */ + // log("xSectorSize(",pFile,")"); + // return 4096 /* ==> SQLITE_DEFAULT_SECTOR_SIZE */; + //}) + + const rc = capi.sqlite3_vfs_register(oVfs.pointer, 0); + if(rc){ + oVfs.dispose(); + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + capi.sqlite3_vfs_register.addReference(oVfs, oIom); + warn("End of (very incomplete) OPFS setup.", oVfs); + //oVfs.dispose()/*only because we can't yet do anything with it*/; + +}/*initOpfsBits()*/; + +(async function(){ + importScripts('sqlite3.js'); + + const test1 = function(db){ + db.exec("create table if not exists t(a);") + .transaction(function(db){ + db.prepare("insert into t(a) values(?)") + .bind(new Date().getTime()) + .stepFinalize(); + stdout("Number of values in table t:", + db.selectValue("select count(*) from t")); + }); + }; + + const runTests = async function(Module){ + //stdout("Module",Module); + self._MODULE = Module /* this is only to facilitate testing from the console */; + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + + if(1){ + let errCount = 0; + [ + 'FileSystemHandle', 'FileSystemFileHandle', 'FileSystemDirectoryHandle', + 'FileSystemSyncAccessHandle' + ].forEach(function(n){ + const f = self[n]; + if(f){ + warn(n,f); + warn(n+'.prototype',f.prototype); + }else{ + stderr("MISSING",n); + ++errCount; + } + }); + if(errCount) return; + } + warn('self',self); + await initOpfsBits(sqlite3); + + if(1) return; + + let persistentDir; + if(1){ + persistentDir = ''; + }else{ + persistentDir = capi.sqlite3_web_persistent_dir(); + if(persistentDir){ + stderr("Persistent storage dir:",persistentDir); + }else{ + stderr("No persistent storage available."); + return; + } + } + const startTime = performance.now(); + let db; + try { + db = new oo.DB(persistentDir+'/foo.db'); + stdout("DB filename:",db.filename,db.fileName()); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + test1 + ].forEach((f)=>{ + const n = performance.now(); + stdout(banner1,"Running",f.name+"()..."); + f(db, sqlite3, Module); + stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); + }); + }finally{ + if(db) db.close(); + } + stdout("Total test time:",(performance.now() - startTime),"ms"); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(runTests); +})(); |