diff options
Diffstat (limited to 'ext/wasm/api')
| -rw-r--r-- | ext/wasm/api/README.md | 8 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-glue.js | 13 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-oo1.js | 19 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-prologue.js | 102 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-worker1.js (renamed from ext/wasm/api/sqlite3-api-worker.js) | 221 |
5 files changed, 187 insertions, 176 deletions
diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 43d2b0dd5..9000697b2 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -60,17 +60,17 @@ browser client: high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker.js`\ +- `sqlite3-api-worker1.js`\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker.js`\ + - `sqlite3-worker1.js`\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module - and runs the Worker API which is implemented in - `sqlite3-api-worker.js`. + and runs the Worker #1 API which is implemented in + `sqlite3-api-worker1.js`. - `sqlite3-api-opfs.js`\ is an in-development/experimental sqlite3 VFS wrapper, the goal of which being to use Google Chrome's Origin-Private FileSystem (OPFS) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e962c93b6..efcd6fea9 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -128,7 +128,7 @@ */ __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', "int", ["sqlite3*", "string", - "int"/*MUST always be negative*/, + "int"/*ignored for this impl!*/, "int", "**", "**"/*MUST be 0 or null or undefined!*/]); /** @@ -148,19 +148,10 @@ /* Documented in the api object's initializer. */ capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ - /* 2022-07-08: xWrap() 'string' arg handling may be able do this - special-case handling for us. It needs to be tested. Or maybe - not: we always want to treat pzTail as null when passed a - non-pointer SQL string and the argument adapters don't have - enough state to know that. Maybe they could/should, by passing - the currently-collected args as an array as the 2nd arg to the - argument adapters? Or maybe we collect all args in an array, - pass that to an optional post-args-collected callback, and give - it a chance to manipulate the args before we pass them on? */ if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); switch(typeof sql){ case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); - case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); + case 'number': return __prepare.full(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail); default: return util.sqlite3_wasm_db_error( pDb, capi.SQLITE_MISUSE, diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index e54e27ec2..9efc38b91 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -322,15 +322,14 @@ } }, /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. + Similar to this.filename but will return NULL for special names + like ":memory:". Not of much use until we have filesystem + support. Throws if the DB has been closed. If passed an + argument it then it will return the filename of the ATTACHEd db + with that name, else it assumes a name of `main`. */ - fileName: function(dbName){ - return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); + fileName: function(dbName='main'){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); }, /** Returns true if this db instance has a name which resolves to a @@ -759,7 +758,7 @@ capi.sqlite3_result_null(pCx); break; }else if(util.isBindableTypedArray(val)){ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); capi.wasm.dealloc(pBlob); @@ -1072,7 +1071,7 @@ capi.wasm.scopedAllocPop(stack); } }else{ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); try{ rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 08e3ebf6e..cf9fd896a 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -173,36 +173,6 @@ self.sqlite3ApiBootstrap = function(config){ */ const capi = { /** - An Error subclass which is thrown by this object's alloc() method - on OOM. - */ - WasmAllocError: WasmAllocError, - /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). - - That said, very few cases in the API can result in - client-defined functions propagating exceptions via the C-style - API. Most notably, this applies ot User-defined SQL Functions - (UDFs) registered via sqlite3_create_function_v2(). For that - specific case it is recommended that all UDF creation be - funneled through a utility function and that a wrapper function - be added around the UDF which catches any exception and sets - the error state to OOM. (The overall complexity of registering - UDFs essentially requires a helper for doing so!) - */ - alloc: undefined/*installed later*/, - /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). - */ - dealloc: undefined/*installed later*/, - /** When using sqlite3_open_v2() it is important to keep the following in mind: @@ -365,6 +335,33 @@ self.sqlite3ApiBootstrap = function(config){ || toss("API config object requires a WebAssembly.Memory object", "in either config.exports.memory (exported)", "or config.memory (imported)."), + + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + That said, very few cases in the API can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies ot User-defined SQL Functions + (UDFs) registered via sqlite3_create_function_v2(). For that + specific case it is recommended that all UDF creation be + funneled through a utility function and that a wrapper function + be added around the UDF which catches any exception and sets + the error state to OOM. (The overall complexity of registering + UDFs essentially requires a helper for doing so!) + */ + alloc: undefined/*installed later*/, + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + */ + dealloc: undefined/*installed later*/ + /* Many more wasm-related APIs get installed later on. */ }/*wasm*/ }/*capi*/; @@ -387,7 +384,7 @@ self.sqlite3ApiBootstrap = function(config){ Int8Array types and will throw if srcTypedArray is of any other type. */ - capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + capi.wasm.allocFromTypedArray = function(srcTypedArray){ affirmBindableTypedArray(srcTypedArray); const pRet = this.alloc(srcTypedArray.byteLength || 1); this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); @@ -400,11 +397,13 @@ self.sqlite3ApiBootstrap = function(config){ const f = capi.wasm.exports[key]; if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); } + capi.wasm.alloc = function(n){ const m = this.exports[keyAlloc](n); if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); return m; }.bind(capi.wasm) + capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); /** @@ -554,7 +553,8 @@ self.sqlite3ApiBootstrap = function(config){ ["sqlite3_value_text", "string", "*"], ["sqlite3_value_type", "int", "*"], ["sqlite3_vfs_find", "*", "string"], - ["sqlite3_vfs_register", "int", "*", "int"] + ["sqlite3_vfs_register", "int", "*", "int"], + ["sqlite3_wasm_vfs_unlink", "int", "string"] ]/*capi.wasm.bindingSignatures*/; if(false && capi.wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ @@ -583,18 +583,26 @@ self.sqlite3ApiBootstrap = function(config){ If the wasm environment has a persistent storage directory, its path is returned by this function. If it does not then - it returns one of: + it returns "" (noting that "" is a falsy value). - - `undefined` if initIfNeeded is false and this function has - never been called before. + The first time this is called, this function inspects the current + environment to determine whether persistence filesystem support + is available and, if it is, enables it (if needed). - - `""` if no persistent storage is available. + TODOs and caveats: - Note that in both cases the return value is falsy. + - The directory name (mount point) for persistent storage is + currently hard-coded. It needs to be configurable. + + - If persistent storage is available at the root of the virtual + filesystem, this interface cannot currently distinguish that + from the lack of persistence. That case cannot currently (with + WASMFS/OPFS) happen, but it is conceivably possible in future + environments or non-browser runtimes (none of which are yet + supported targets). */ - capi.sqlite3_web_persistent_dir = function(initIfNeeded=true){ + capi.sqlite3_web_persistent_dir = function(){ if(undefined !== __persistentDir) return __persistentDir; - else if(!initIfNeeded) return; // If we have no OPFS, there is no persistent dir if(!self.FileSystemHandle || !self.FileSystemDirectoryHandle || !self.FileSystemFileHandle){ @@ -625,8 +633,22 @@ self.sqlite3ApiBootstrap = function(config){ } }.bind(capi); + /** + Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a + non-empty string and the given name has that string as its + prefix, else returns false. + */ + capi.sqlite3_web_filename_is_persistent = function(name){ + const p = this.sqlite3_web_persistent_dir(); + return (p && name) ? name.startsWith(p) : false; + }.bind(capi); + /* The remainder of the API will be set up in later steps. */ return { + /** + An Error subclass which is thrown by this.wasm.alloc() on OOM. + */ + WasmAllocError: WasmAllocError, capi, postInit: [ /* some pieces of the API may install functions into this array, @@ -635,8 +657,6 @@ self.sqlite3ApiBootstrap = function(config){ the current global object and sqlite3 is the object returned from sqlite3ApiBootstrap(). This array will be removed at the end of the API setup process. */], - /** Config is needed downstream for gluing pieces together. It - will be removed at the end of the API setup process. */ config }; }/*sqlite3ApiBootstrap()*/; diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker1.js index 95b27b21e..565946bbc 100644 --- a/ext/wasm/api/sqlite3-api-worker.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -10,26 +10,40 @@ *********************************************************************** - This file implements a Worker-based wrapper around SQLite3 OO API - #1. + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + This function implements a Worker-based wrapper around SQLite3 OO + API #1, colloquially known as "Worker API #1". In order to permit this API to be loaded in worker threads without automatically registering onmessage handlers, initializing the - worker API requires calling initWorkerAPI(). If this function + worker API requires calling initWorker1API(). If this function is called from a non-worker thread then it throws an exception. - When initialized, it installs message listeners to receive messages - from the main thread and then it posts a message in the form: + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: ``` - {type:'sqlite3-api',data:'worker-ready'} + {type:'sqlite3-api',data:'worker1-ready'} ``` - This file requires that the core C-style sqlite3 API and OO API #1 - have been loaded and that self.sqlite3 contains both, - as documented for those APIs. + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. */ -self.sqlite3.initWorkerAPI = function(){ +self.sqlite3.initWorker1API = function(){ 'use strict'; /** UNDER CONSTRUCTION @@ -39,39 +53,12 @@ self.sqlite3.initWorkerAPI = function(){ cannot pass callback functions between the window thread and a worker thread, so we have to receive all db results via asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who wants - to juggle six database handles at once. Do we add that complexity - or tell such users to write their own code using the provided - lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we collect - all results and bundle them back in a single message? The former - is, generically speaking, more memory-efficient but the latter - far easier to implement in this environment. The latter is - untennable for large data sets. Despite a web page hypothetically - being a relatively limited environment, there will always be - those users who feel that they should/need to be able to work - with multi-hundred-meg (or larger) blobs, and passing around - arrays of those may quickly exhaust the JS engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB.pointer (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB.pointer) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. + with a distinctly different shape than OO API #1. + + TODOs include, but are not necessarily limited to: + + - Support for handling multiple DBs via this interface is under + development. - Revisit how virtual files are managed. We currently delete DBs from the virtual filesystem when we close them, for the sake of @@ -79,6 +66,8 @@ self.sqlite3.initWorkerAPI = function(){ require that we give up that habit. Similarly, fully supporting ATTACH, where a user can upload multiple DBs and ATTACH them, also requires the that we manage the VFS entries better. + Related: we most definitely do not want to delete persistent DBs + (e.g. stored on OPFS) when they're closed. */ const toss = (...args)=>{throw new Error(args.join(' '))}; if('function' !== typeof importScripts){ @@ -116,8 +105,7 @@ self.sqlite3.initWorkerAPI = function(){ // same filename and close/reopen it (or just pass it back as is?). if(!arg && this.defaultDb) return this.defaultDb; //???if(this.defaultDb) this.defaultDb.close(); - let db; - db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); + const db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); this.dbs[getDbId(db)] = db; if(!this.defaultDb) this.defaultDb = db; return db; @@ -125,13 +113,17 @@ self.sqlite3.initWorkerAPI = function(){ close: function(db,alsoUnlink){ if(db){ delete this.dbs[getDbId(db)]; - db.close(alsoUnlink); + const filename = db.fileName(); + db.close(); if(db===this.defaultDb) this.defaultDb = undefined; + if(alsoUnlink && filename){ + sqlite3.capi.sqlite3_wasm_vfs_unlink(filename); + } } }, post: function(type,data,xferList){ if(xferList){ - self.postMessage({type, data},xferList); + self.postMessage( {type, data}, xferList ); xferList.length = 0; }else{ self.postMessage({type, data}); @@ -173,6 +165,68 @@ self.sqlite3.initWorkerAPI = function(){ const wMsgHandler = { xfer: [/*Temp holder for "transferable" postMessage() state.*/], /** + Proxy for the DB constructor. Expects to be passed a single + object or a falsy value to use defaults. The object may + have a filename property to name the db file (see the DB + constructor for peculiarities and transformations) and/or a + buffer property (a Uint8Array holding a complete database + file's contents). The response is an object: + + { + filename: db filename (possibly differing from the input), + + dbId: an opaque ID value which must be passed to other calls + in this API to tell them which db to use. If it is not + provided to future calls, they will default to + operating on the first-opened db. + + messageId: if the client-sent message included this field, + it is mirrored in the response. + } + */ + open: function(ev){ + const args = [], data = (ev.data || {}); + if(data.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + if(data.filename) args.push(data.filename); + const db = wState.open(args); + return { + filename: db.filename, + dbId: getDbId(db) + }; + }, + /** + Proxy for DB.close(). If ev.data may either be a boolean or + an object with an `unlink` property. If that value is + truthy then the db file (if the db is currently open) will + be unlinked from the virtual filesystem, else it will be + kept intact. The response object is: + + { + filename: db filename _if_ the db is opened when this + is called, else the undefined value, + unlink: boolean. If true, unlink() (delete) the db file + after closing int. Any error while deleting it is + ignored. + } + + It does not error if the given db is already closed or no db is + provided. It is simply does nothing useful in that case. + */ + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename, + dbId: db ? getDbId(db) : undefined + }; + if(db){ + wState.close(db, !!((ev.data && 'object'===typeof ev.data) + ? ev.data.unlink : false)); + } + return response; + }, + /** Proxy for DB.exec() which expects a single argument of type string (SQL to execute) or an options object in the form expected by exec(). The notable differences from exec() @@ -266,10 +320,16 @@ self.sqlite3.initWorkerAPI = function(){ exports of ":memory:" and "" (temp file) DBs. The latter is ostensibly easy because the file is (potentially) on disk, but the former does not have a structure which maps directly to a - db file image. + db file image. We can VACUUM INTO a :memory:/temp db into a + file for that purpose, though. */ export: function(ev){ toss("export() requires reimplementing for portability reasons."); + /** + We need to reimplement this to use the Emscripten FS + interface. That part used to be in the OO#1 API but that + dependency was removed from that level of the API. + */ /**const db = getMsgDb(ev); const response = { buffer: db.exportBinaryImage(), @@ -279,66 +339,6 @@ self.sqlite3.initWorkerAPI = function(){ this.xfer.push(response.buffer.buffer); return response;**/ }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input), - - id: an opaque ID value intended for future distinction - between multiple db handles. Messages including a specific - ID will use the DB for that ID. - - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.simulateError){ - toss("Throwing because of open.simulateError flag."); - } - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return { - filename: db.filename, - dbId: getDbId(db) - }; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - { - filename: db filename _if_ the db is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const db = getMsgDb(ev,false); - const response = { - filename: db && db.filename - }; - if(db){ - wState.close(db, !!((ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data)); - } - return response; - }, toss: function(ev){ toss("Testing worker exception"); } @@ -352,7 +352,8 @@ self.sqlite3.initWorkerAPI = function(){ { type: apiCommand, dbId: optional DB ID value (else uses a default db handle) - data: apiArguments + data: apiArguments, + messageId: optional client-specific value } As a rule, these commands respond with a postMessage() of their @@ -416,5 +417,5 @@ self.sqlite3.initWorkerAPI = function(){ response.departureTime = ev.departureTime; wState.post(evType, response, wMsgHandler.xfer); }; - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); + setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker1-ready'}), 0); }.bind({self, sqlite3: self.sqlite3}); |