diff options
Diffstat (limited to 'ext/wasm/api')
| -rw-r--r-- | ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 2 | ||||
| -rw-r--r-- | ext/wasm/api/README.md | 8 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-cleanup.js | 67 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-glue.js | 39 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-oo1.js | 655 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-opfs.js | 9 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-prologue.js | 308 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-worker.js | 420 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-worker1.js | 621 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-wasm.c | 118 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-worker.js | 31 |
11 files changed, 1456 insertions, 822 deletions
diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 8f103c7c0..aead79e50 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -66,7 +66,5 @@ _sqlite3_value_text _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register -_sqlite3_wasm_db_error -_sqlite3_wasm_enum_json _malloc _free diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 43d2b0dd5..9000697b2 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -60,17 +60,17 @@ browser client: high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker.js`\ +- `sqlite3-api-worker1.js`\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker.js`\ + - `sqlite3-worker1.js`\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module - and runs the Worker API which is implemented in - `sqlite3-api-worker.js`. + and runs the Worker #1 API which is implemented in + `sqlite3-api-worker1.js`. - `sqlite3-api-opfs.js`\ is an in-development/experimental sqlite3 VFS wrapper, the goal of which being to use Google Chrome's Origin-Private FileSystem (OPFS) diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index a2f921a5d..1b57cdc5d 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -11,34 +11,45 @@ *********************************************************************** This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other files so that it can clean - up any global systems temporarily used for setting up the API's - various subsystems. + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. */ 'use strict'; -self.sqlite3.postInit.forEach( - self.importScripts/*global is a Worker*/ - ? function(f){ - /** We try/catch/report for the sake of failures which happen in - a Worker, as those exceptions can otherwise get completely - swallowed, leading to confusing downstream errors which have - nothing to do with this failure. */ - try{ f(self, self.sqlite3) } - catch(e){ - console.error("Error in postInit() function:",e); - throw e; - } - } - : (f)=>f(self, self.sqlite3) -); -delete self.sqlite3.postInit; -if(self.location && +self.location.port > 1024){ - console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); - self.S = self.sqlite3; +if('undefined' !== typeof Module){ // presumably an Emscripten build + /** + Install a suitable default configuration for sqlite3ApiBootstrap(). + */ + const SABC = self.sqlite3ApiBootstrap.defaultConfig; + SABC.Module = Module /* ==> Currently needs to be exposed here for test code. NOT part + of the public API. */; + SABC.exports = Module['asm']; + SABC.memory = Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */; + + /** + For current (2022-08-22) purposes, automatically call + sqlite3ApiBootstrap(). That decision will be revisited at some + point, as we really want client code to be able to call this to + configure certain parts. Clients may modify + self.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(). + */ + //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); + const sqlite3 = self.sqlite3ApiBootstrap(); + delete self.sqlite3ApiBootstrap; + + if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); + self.S = sqlite3; + } + + /* Clean up temporary references to our APIs... */ + delete sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; + //console.warn("Module.sqlite3 =",Module.sqlite3); + Module.sqlite3 = sqlite3 /* Currently needed by test code and sqlite3-worker1.js */; +}else{ + console.warn("This is not running in an Emscripten module context, so", + "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "of config info for the WASM environment.", + "It must be called manually."); } -/* Clean up temporary global-scope references to our APIs... */ -self.sqlite3.config.Module.sqlite3 = self.sqlite3 -/* ^^^^ Currently needed by test code and Worker API setup */; -delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; -delete self.sqlite3 /* clean up our global-scope reference */; -//console.warn("Module.sqlite3 =",Module.sqlite3); diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e962c93b6..3a9e8803c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -16,23 +16,9 @@ initializes the main API pieces so that the downstream components (e.g. sqlite3-api-oo1.js) have all that they need. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - - self.sqlite3 = self.sqlite3ApiBootstrap({ - Module: Module /* ==> Emscripten-style Module object. Currently - needs to be exposed here for test code. NOT part - of the public API. */, - exports: Module['asm'], - memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */, - bigIntEnabled: !!self.BigInt64Array, - allocExportName: 'malloc', - deallocExportName: 'free' - }); - delete self.sqlite3ApiBootstrap; - - const sqlite3 = self.sqlite3; const capi = sqlite3.capi, wasm = capi.wasm, util = capi.util; self.WhWasmUtilInstaller(capi.wasm); delete self.WhWasmUtilInstaller; @@ -57,7 +43,7 @@ return oldP(v); }; wasm.xWrap.argAdapter('.pointer', adapter); - } + } /* ".pointer" xWrap() argument adapter */ // WhWasmUtil.xWrap() bindings... { @@ -77,8 +63,11 @@ for(const e of wasm.bindingSignatures){ capi[e[0]] = wasm.xWrap.apply(null, e); } + for(const e of wasm.bindingSignatures.wasm){ + capi.wasm[e[0]] = wasm.xWrap.apply(null, e); + } - /* For functions which cannot work properly unless + /* For C API functions which cannot work properly unless wasm.bigIntEnabled is true, install a bogus impl which throws if called when bigIntEnabled is false. */ const fI64Disabled = function(fname){ @@ -128,7 +117,7 @@ */ __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', "int", ["sqlite3*", "string", - "int"/*MUST always be negative*/, + "int"/*ignored for this impl!*/, "int", "**", "**"/*MUST be 0 or null or undefined!*/]); /** @@ -148,19 +137,10 @@ /* Documented in the api object's initializer. */ capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ - /* 2022-07-08: xWrap() 'string' arg handling may be able do this - special-case handling for us. It needs to be tested. Or maybe - not: we always want to treat pzTail as null when passed a - non-pointer SQL string and the argument adapters don't have - enough state to know that. Maybe they could/should, by passing - the currently-collected args as an array as the 2nd arg to the - argument adapters? Or maybe we collect all args in an array, - pass that to an optional post-args-collected callback, and give - it a chance to manipulate the args before we pass them on? */ if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); switch(typeof sql){ case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); - case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); + case 'number': return __prepare.full(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail); default: return util.sqlite3_wasm_db_error( pDb, capi.SQLITE_MISUSE, @@ -207,5 +187,4 @@ capi[s.name] = sqlite3.StructBinder(s); } } - -})(self); +}); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 9e5473396..af179d1fe 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -14,10 +14,9 @@ WASM build. It requires that sqlite3-api-glue.js has already run and it installs its deliverable as self.sqlite3.oo1. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = (...args)=>{throw new Error(args.join(' '))}; - const sqlite3 = self.sqlite3 || toss("Missing main sqlite3 object."); const capi = sqlite3.capi, util = capi.util; /* What follows is colloquially known as "OO API #1". It is a binding of the sqlite3 API which is designed to be run within @@ -59,14 +58,86 @@ enabling clients to unambiguously identify such exceptions. */ class SQLite3Error extends Error { + /** + Constructs this object with a message equal to all arguments + concatenated with a space between each one. + */ constructor(...args){ - super(...args); + super(args.join(' ')); this.name = 'SQLite3Error'; } }; - const toss3 = (...args)=>{throw new SQLite3Error(args)}; + const toss3 = (...args)=>{throw new SQLite3Error(...args)}; sqlite3.SQLite3Error = SQLite3Error; + // Documented in DB.checkRc() + const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + throw new SQLite3Error( + "sqlite result code",sqliteResultCode+":", + (dbPtr + ? capi.sqlite3_errmsg(dbPtr) + : capi.sqlite3_errstr(sqliteResultCode)) + ); + } + }; + + /** + A proxy for DB class constructors. It must be called with the + being-construct DB object as its "this". + */ + const dbCtorHelper = function ctor(fn=':memory:', flags='c', vfsName){ + if(!ctor._name2vfs){ + // Map special filenames which we handle here (instead of in C) + // to some helpful metadata... + ctor._name2vfs = Object.create(null); + const isWorkerThread = (self.window===self /*===running in main window*/) + ? false + : (n)=>toss3("The VFS for",n,"is only available in the main window thread.") + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'session') + }; + } + if('string'!==typeof fn){ + toss3("Invalid filename for DB constructor."); + } + const vfsCheck = ctor._name2vfs[fn]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = vfsCheck.filename(fn); + } + let ptr, oflags = 0; + if( flags.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flags.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = capi.wasm.scopedAllocPush(); + try { + const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; + const pVfsName = vfsName ? capi.wasm.scopedAllocCString(vfsName) : 0; + const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName); + ptr = capi.wasm.getPtrValue(ppDb); + checkSqlite3Rc(ptr, rc); + }catch( e ){ + if( ptr ) capi.sqlite3_close_v2(ptr); + throw e; + }finally{ + capi.wasm.scopedAllocPop(stack); + } + this.filename = fn; + __ptrMap.set(this, ptr); + __stmtMap.set(this, Object.create(null)); + __udfMap.set(this, Object.create(null)); + }; + /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. @@ -80,39 +151,48 @@ not resolve to real filenames, but "" uses an on-storage temporary database and requires that the VFS support that. - The db is currently opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API will change in the future - permit the caller to provide those flags via an additional - argument. + The second argument specifies the open/create mode for the + database. It must be string containing a sequence of letters (in + any order, but case sensitive) specifying the mode: + + - "c" => create if it does not exist, else fail if it does not + exist. Implies the "w" flag. + + - "w" => write. Implies "r": a db cannot be write-only. + + - "r" => read-only if neither "w" nor "c" are provided, else it + is ignored. + + If "w" is not provided, the db is implicitly read-only, noting that + "rc" is meaningless + + Any other letters are currently ignored. The default is + "c". These modes are ignored for the special ":memory:" and "" + names. + + The final argument is analogous to the final argument of + sqlite3_open_v2(): the name of an sqlite3 VFS. Pass a falsy value, + or not at all, to use the default. If passed a value, it must + be the string name of a VFS For purposes of passing a DB instance to C-style sqlite3 - functions, its read-only `pointer` property holds its `sqlite3*` - pointer value. That property can also be used to check whether - this DB instance is still open. + functions, the DB object's read-only `pointer` property holds its + `sqlite3*` pointer value. That property can also be used to check + whether this DB instance is still open. + + + EXPERIMENTAL: in the main window thread, the filenames + ":localStorage:" and ":sessionStorage:" are special: they cause + the db to use either localStorage or sessionStorage for storing + the database. In this mode, only a single database is permitted + in each storage object. This feature is experimental and subject + to any number of changes (including outright removal). This + support requires a specific build of sqlite3, the existence of + which can be determined at runtime by checking for a non-0 return + value from sqlite3.capi.sqlite3_vfs_find("kvvfs"). */ - const DB = function ctor(fn=':memory:'){ - if('string'!==typeof fn){ - toss3("Invalid filename for DB constructor."); - } - const stack = capi.wasm.scopedAllocPush(); - let ptr; - try { - const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; - const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE - | capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_EXRESCODE, null); - ptr = capi.wasm.getMemValue(ppDb, '*'); - ctor.checkRc(ptr, rc); - }catch(e){ - if(ptr) capi.sqlite3_close_v2(ptr); - throw e; - } - finally{capi.wasm.scopedAllocPop(stack);} - this.filename = fn; - __ptrMap.set(this, ptr); - __stmtMap.set(this, Object.create(null)); - __udfMap.set(this, Object.create(null)); + const DB = function ctor(fn=':memory:', flags='c', vfsName){ + dbCtorHelper.apply(this, Array.prototype.slice.call(arguments)); }; /** @@ -141,6 +221,15 @@ For purposes of passing a Stmt instance to C-style sqlite3 functions, its read-only `pointer` property holds its `sqlite3_stmt*` pointer value. + + Other non-function properties include: + + - `db`: the DB object which created the statement. + + - `columnCount`: the number of result columns in the query, or 0 for + queries which cannot return results. + + - `parameterCount`: the number of bindable paramters in the query. */ const Stmt = function(){ if(BindTypes!==arguments[2]){ @@ -163,7 +252,7 @@ Reminder: this will also fail after the statement is finalized but the resulting error will be about an out-of-bounds column - index. + index rather than a statement-is-finalized error. */ const affirmColIndex = function(stmt,ndx){ if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ @@ -173,16 +262,20 @@ }; /** - Expects to be passed (arguments) from DB.exec() and - DB.execMulti(). Does the argument processing/validation, throws - on error, and returns a new object on success: + Expects to be passed the `arguments` object from DB.exec(). Does + the argument processing/validation, throws on error, and returns + a new object on success: { sql: the SQL, opt: optionsObj, cbArg: function} - cbArg is only set if the opt.callback is set, in which case - it's a function which expects to be passed the current Stmt - and returns the callback argument of the type indicated by - the input arguments. + The opt object is a normalized copy of any passed to this + function. The sql will be converted to a string if it is provided + in one of the supported non-string formats. + + cbArg is only set if the opt.callback or opt.resultRows are set, + in which case it's a function which expects to be passed the + current Stmt and returns the callback argument of the type + indicated by the input arguments. */ const parseExecArgs = function(args){ const out = Object.create(null); @@ -194,6 +287,8 @@ }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; out.sql = out.opt.sql; + }else if(Array.isArray(args[0])){ + out.sql = args[0]; } break; case 2: @@ -211,14 +306,14 @@ } if(out.opt.callback || out.opt.resultRows){ switch((undefined===out.opt.rowMode) - ? 'stmt' : out.opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get({}); break; + ? 'array' : out.opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; case 'array': out.cbArg = (stmt)=>stmt.get([]); break; case 'stmt': if(Array.isArray(out.opt.resultRows)){ - toss3("Invalid rowMode for resultRows array: must", + toss3("exec(): invalid rowMode for a resultRows array: must", "be one of 'array', 'object',", - "or a result column number."); + "a result column number, or column name reference."); } out.cbArg = (stmt)=>stmt; break; @@ -226,6 +321,19 @@ if(util.isInt32(out.opt.rowMode)){ out.cbArg = (stmt)=>stmt.get(out.opt.rowMode); break; + }else if('string'===typeof out.opt.rowMode && out.opt.rowMode.length>1){ + /* "$X", ":X", and "@X" fetch column named "X" (case-sensitive!) */ + const prefix = out.opt.rowMode[0]; + if(':'===prefix || '@'===prefix || '$'===prefix){ + out.cbArg = function(stmt){ + const rc = stmt.get(this.obj)[this.colName]; + return (undefined===rc) ? toss3("exec(): unknown result column:",this.colName) : rc; + }.bind({ + obj:Object.create(null), + colName: out.opt.rowMode.substr(1) + }); + break; + } } toss3("Invalid rowMode:",out.opt.rowMode); } @@ -234,24 +342,17 @@ }; /** - Expects to be given a DB instance or an `sqlite3*` pointer, and an - sqlite3 API result code. If the result code is not falsy, this - function throws an SQLite3Error with an error message from - sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's - passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." The - various non-0 non-error codes need to be checked for in client - code where they are expected. + Expects to be given a DB instance or an `sqlite3*` pointer (may + be null) and an sqlite3 API result code. If the result code is + not falsy, this function throws an SQLite3Error with an error + message from sqlite3_errmsg(), using dbPtr as the db handle, or + sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a + non-error code like SQLITE_ROW or SQLITE_DONE, it will still + throw but the error string might be "Not an error." The various + non-0 non-error codes need to be checked for in + client code where they are expected. */ - DB.checkRc = function(dbPtr, sqliteResultCode){ - if(sqliteResultCode){ - if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; - throw new SQLite3Error([ - "sqlite result code",sqliteResultCode+":", - capi.sqlite3_errmsg(dbPtr) || "Unknown db error." - ].join(' ')); - } - }; + DB.checkRc = checkSqlite3Rc; DB.prototype = { /** @@ -300,26 +401,24 @@ } }, /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. + Similar to this.filename but will return NULL for special names + like ":memory:". Not of much use until we have filesystem + support. Throws if the DB has been closed. If passed an + argument it then it will return the filename of the ATTACHEd db + with that name, else it assumes a name of `main`. */ - fileName: function(dbName){ - return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); + fileName: function(dbName='main'){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); }, /** Returns true if this db instance has a name which resolves to a file. If the name is "" or ":memory:", it resolves to false. Note that it is not aware of the peculiarities of URI-style names and a URI-style name for a ":memory:" db will fool it. + Returns false if this db is closed. */ hasFilename: function(){ - const fn = this.filename; - if(!fn || ':memory'===fn) return false; - return true; + return this.filename && ':memory'!==this.filename; }, /** Returns the name of the given 0-based db number, as documented @@ -343,9 +442,8 @@ required to check `stmt.pointer` after calling `prepare()` in order to determine whether the Stmt instance is empty or not. Long-time practice (with other sqlite3 script bindings) - suggests that the empty-prepare case is sufficiently rare (and - useless) that supporting it here would simply hurt overall - usability. + suggests that the empty-prepare case is sufficiently rare that + supporting it here would simply hurt overall usability. */ prepare: function(sql){ affirmDbOpen(this); @@ -354,7 +452,7 @@ try{ ppStmt = capi.wasm.scopedAllocPtr()/* output (sqlite3_stmt**) arg */; DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); - pStmt = capi.wasm.getMemValue(ppStmt, '*'); + pStmt = capi.wasm.getPtrValue(ppStmt); } finally {capi.wasm.scopedAllocPop(stack)} if(!pStmt) toss3("Cannot prepare empty SQL."); @@ -363,70 +461,6 @@ return stmt; }, /** - This function works like execMulti(), and takes most of the - same arguments, but is more efficient (performs much less - work) when the input SQL is only a single statement. If - passed a multi-statement SQL, it only processes the first - one. - - This function supports the following additional options not - supported by execMulti(): - - - .multi: if true, this function acts as a proxy for - execMulti() and behaves identically to that function. - - - .columnNames: if this is an array and the query has - result columns, the array is passed to - Stmt.getColumnNames() to append the column names to it - (regardless of whether the query produces any result - rows). If the query has no result columns, this value is - unchanged. - - The following options to execMulti() are _not_ supported by - this method (they are simply ignored): - - - .saveSql - */ - exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ - affirmDbOpen(this); - const arg = parseExecArgs(arguments); - if(!arg.sql) return this; - else if(arg.opt.multi){ - return this.execMulti(arg, undefined, BindTypes); - } - const opt = arg.opt; - let stmt, rowTarget; - try { - if(Array.isArray(opt.resultRows)){ - rowTarget = opt.resultRows; - } - stmt = this.prepare(arg.sql); - if(stmt.columnCount && Array.isArray(opt.columnNames)){ - stmt.getColumnNames(opt.columnNames); - } - if(opt.bind) stmt.bind(opt.bind); - if(opt.callback || rowTarget){ - while(stmt.step()){ - const row = arg.cbArg(stmt); - if(rowTarget) rowTarget.push(row); - if(opt.callback){ - stmt._isLocked = true; - opt.callback(row, stmt); - stmt._isLocked = false; - } - } - }else{ - stmt.step(); - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - } - return this; - }/*exec()*/, - /** Executes one or more SQL statements in the form of a single string. Its arguments must be either (sql,optionsObject) or (optionsObject). In the latter case, optionsObject.sql @@ -440,92 +474,113 @@ The optional options object may contain any of the following properties: - - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string, Uint8Array, or an - array of strings (in which case they're concatenated - together as-is, with no separator between elements, - before evaluation). - - - .bind = a single value valid as an argument for - Stmt.bind(). This is ONLY applied to the FIRST non-empty - statement in the SQL which has any bindable - parameters. (Empty statements are skipped entirely.) - - - .callback = a function which gets called for each row of - the FIRST statement in the SQL which has result - _columns_, but only if that statement has any result - _rows_. The second argument passed to the callback is - always the current Stmt object (so that the caller may - collect column names, or similar). The first argument - passed to the callback defaults to the current Stmt - object but may be changed with ... - - - .rowMode = either a string describing what type of argument - should be passed as the first argument to the callback or an - integer representing a result column index. A `rowMode` of - 'object' causes the results of `stmt.get({})` to be passed to - the `callback` and/or appended to `resultRows`. A value of - 'array' causes the results of `stmt.get([])` to be passed to - passed on. A value of 'stmt' is equivalent to the default, - passing the current Stmt to the callback (noting that it's - always passed as the 2nd argument), but this mode will trigger - an exception if `resultRows` is an array. If `rowMode` is an - integer, only the single value from that result column will be - passed on. Any other value for the option triggers an - exception. + - `.sql` = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an array + of strings. In the latter case they're concatenated together + as-is, _with no separator_ between elements, before evaluation. + The array form is often simpler for long hand-written queries. + + - `.bind` = a single value valid as an argument for + Stmt.bind(). This is _only_ applied to the _first_ non-empty + statement in the SQL which has any bindable parameters. (Empty + statements are skipped entirely.) - - .resultRows: if this is an array, it functions similarly to - the `callback` option: each row of the result set (if any) of - the FIRST first statement which has result _columns_ is - appended to the array in the format specified for the `rowMode` - option, with the exception that the only legal values for - `rowMode` in this case are 'array' or 'object', neither of - which is the default. It is legal to use both `resultRows` and - `callback`, but `resultRows` is likely much simpler to use for - small data sets and can be used over a WebWorker-style message - interface. execMulti() throws if `resultRows` is set and - `rowMode` is 'stmt' (which is the default!). - - - saveSql = an optional array. If set, the SQL of each + - `.saveSql` = an optional array. If set, the SQL of each executed statement is appended to this array before the - statement is executed (but after it is prepared - we - don't have the string until after that). Empty SQL - statements are elided. - - See also the exec() method, which is a close cousin of this - one. - - ACHTUNG #1: The callback MUST NOT modify the Stmt - object. Calling any of the Stmt.get() variants, - Stmt.getColumnName(), or similar, is legal, but calling - step() or finalize() is not. Routines which are illegal - in this context will trigger an exception. - - ACHTUNG #2: The semantics of the `bind` and `callback` - options may well change or those options may be removed - altogether for this function (but retained for exec()). - Generally speaking, neither bind parameters nor a callback - are generically useful when executing multi-statement SQL. + statement is executed (but after it is prepared - we don't have + the string until after that). Empty SQL statements are elided. + + ================================================================== + The following options apply _only_ to the _first_ statement + which has a non-zero result column count, regardless of whether + the statement actually produces any result rows. + ================================================================== + + - `.callback` = a function which gets called for each row of + the result set, but only if that statement has any result + _rows_. The callback's "this" is the options object. The second + argument passed to the callback is always the current Stmt + object (so that the caller may collect column names, or + similar). The 2nd argument to the callback is always the Stmt + instance, as it's needed if the caller wants to fetch the + column names or some such (noting that they could also be + fetched via `this.columnNames`, if the client provides the + `columnNames` option). + + ACHTUNG: The callback MUST NOT modify the Stmt object. Calling + any of the Stmt.get() variants, Stmt.getColumnName(), or + similar, is legal, but calling step() or finalize() is + not. Routines which are illegal in this context will trigger an + exception. + + The first argument passed to the callback defaults to an array of + values from the current result row but may be changed with ... + + - `.rowMode` = specifies the type of he callback's first argument. + It may be any of... + + A) A string describing what type of argument should be passed + as the first argument to the callback: + + A.1) `'array'` (the default) causes the results of + `stmt.get([])` to be passed to passed on and/or appended to + `resultRows`. + + A.2) `'object'` causes the results of + `stmt.get(Object.create(null))` to be passed to the + `callback` and/or appended to `resultRows`. Achtung: an SQL + result may have multiple columns with identical names. In + that case, the right-most column will be the one set in this + object! + + A.3) `'stmt'` causes the current Stmt to be passed to the + callback, but this mode will trigger an exception if + `resultRows` is an array because appending the statement to + the array would be unhelpful. + + B) An integer, indicating a zero-based column in the result + row. Only that one single value will be passed on. + + C) A string with a minimum length of 2 and leading character of + ':', '$', or '@' will fetch the row as an object, extract that + one field, and pass that field's value to the callback. Note + that these keys are case-sensitive so must match the case used + in the SQL. e.g. `"select a A from t"` with a `rowMode` of '$A' + would work but '$a' would not. A reference to a column not in + the result set will trigger an exception on the first row (as + the check is not performed until rows are fetched). + + Any other `rowMode` value triggers an exception. + + - `.resultRows`: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any), + with the exception that the `rowMode` 'stmt' is not legal. It + is legal to use both `resultRows` and `callback`, but + `resultRows` is likely much simpler to use for small data sets + and can be used over a WebWorker-style message interface. + exec() throws if `resultRows` is set and `rowMode` is 'stmt'. + + - `.columnNames`: if this is an array, the column names of the + result set are stored in this array before the callback (if + any) is triggered (regardless of whether the query produces any + result rows). If no statement has result columns, this value is + unchanged. Achtung: an SQL result may have multiple columns + with identical names. */ - execMulti: function(/*(sql [,obj]) || (obj)*/){ + exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); const wasm = capi.wasm; - const arg = (BindTypes===arguments[2] - /* ^^^ Being passed on from exec() */ - ? arguments[0] : parseExecArgs(arguments)); - if(!arg.sql) return this; + const arg = parseExecArgs(arguments); + if(!arg.sql){ + return (''===arg.sql) ? this : toss3("exec() requires an SQL string."); + } const opt = arg.opt; const callback = opt.callback; - const resultRows = (Array.isArray(opt.resultRows) + let resultRows = (Array.isArray(opt.resultRows) ? opt.resultRows : undefined); - if(resultRows && 'stmt'===opt.rowMode){ - toss3("rowMode 'stmt' is not valid in combination", - "with a resultRows array."); - } - let rowMode = (((callback||resultRows) && (undefined!==opt.rowMode)) - ? opt.rowMode : undefined); let stmt; let bind = opt.bind; + let evalFirstResult = !!(arg.cbArg || opt.columnNames) /* true to evaluate the first result-returning query */; const stack = wasm.scopedAllocPush(); try{ const isTA = util.isSQLableTypedArray(arg.sql) @@ -544,21 +599,21 @@ if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); - while(wasm.getMemValue(pSql, 'i8') - /* Maintenance reminder: ^^^^ _must_ be i8 or else we + while(pSql && wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we use i32 or similar then we read 4 bytes, read stuff around the NUL terminator, and get stuck in and endless loop at the end of the SQL, endlessly re-preparing an empty statement. */ ){ - wasm.setMemValue(ppStmt, 0, wasm.ptrIR); - wasm.setMemValue(pzTail, 0, wasm.ptrIR); - DB.checkRc(this, capi.sqlite3_prepare_v2( - this.pointer, pSql, sqlByteLen, ppStmt, pzTail + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail )); - const pStmt = wasm.getMemValue(ppStmt, wasm.ptrIR); - pSql = wasm.getMemValue(pzTail, wasm.ptrIR); + const pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); sqlByteLen = pSqlEnd - pSql; if(!pStmt) continue; if(Array.isArray(opt.saveSql)){ @@ -569,28 +624,30 @@ stmt.bind(bind); bind = null; } - if(stmt.columnCount && undefined!==rowMode){ + if(evalFirstResult && stmt.columnCount){ /* Only forward SELECT results for the FIRST query in the SQL which potentially has them. */ - while(stmt.step()){ + evalFirstResult = false; + if(Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + while(!!arg.cbArg && stmt.step()){ stmt._isLocked = true; const row = arg.cbArg(stmt); - if(callback) callback(row, stmt); if(resultRows) resultRows.push(row); + if(callback) callback.apply(opt,[row,stmt]); stmt._isLocked = false; } - rowMode = undefined; }else{ - // Do we need to while(stmt.step()){} here? stmt.step(); } stmt.finalize(); stmt = null; } - }catch(e){ - console.warn("DB.execMulti() is propagating exception",opt,e); + }/*catch(e){ + console.warn("DB.exec() is propagating exception",opt,e); throw e; - }finally{ + }*/finally{ if(stmt){ delete stmt._isLocked; stmt.finalize(); @@ -598,7 +655,7 @@ wasm.scopedAllocPop(stack); } return this; - }/*execMulti()*/, + }/*exec()*/, /** Creates a new scalar UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any @@ -680,8 +737,7 @@ let i, pVal, valType, arg; const tgt = []; for(i = 0; i < argc; ++i){ - pVal = capi.wasm.getMemValue(pArgv + (capi.wasm.ptrSizeof * i), - capi.wasm.ptrIR); + pVal = capi.wasm.getPtrValue(pArgv + (capi.wasm.ptrSizeof * i)); /** Curiously: despite ostensibly requiring 8-byte alignment, the pArgv array is parcelled into chunks of @@ -737,7 +793,7 @@ capi.sqlite3_result_null(pCx); break; }else if(util.isBindableTypedArray(val)){ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); capi.wasm.dealloc(pBlob); @@ -820,6 +876,49 @@ }, /** + Starts a transaction, calls the given callback, and then either + rolls back or commits the savepoint, depending on whether the + callback throws. The callback is passed this db object as its + only argument. On success, returns the result of the + callback. Throws on error. + + Note that transactions may not be nested, so this will throw if + it is called recursively. For nested transactions, use the + savepoint() method or manually manage SAVEPOINTs using exec(). + */ + transaction: function(callback){ + affirmDbOpen(this).exec("BEGIN"); + try { + const rc = callback(this); + this.exec("COMMIT"); + return rc; + }catch(e){ + this.exec("ROLLBACK"); + throw e; + } + }, + + /** + This works similarly to transaction() but uses sqlite3's SAVEPOINT + feature. This function starts a savepoint (with an unspecified name) + and calls the given callback function, passing it this db object. + If the callback returns, the savepoint is released (committed). If + the callback throws, the savepoint is rolled back. If it does not + throw, it returns the result of the callback. + */ + savepoint: function(callback){ + affirmDbOpen(this).exec("SAVEPOINT oo1"); + try { + const rc = callback(this); + this.exec("RELEASE oo1"); + return rc; + }catch(e){ + this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); + throw e; + } + }, + + /** This function currently does nothing and always throws. It WILL BE REMOVED pending other refactoring, to eliminate a hard dependency on Emscripten. This feature will be moved into a @@ -1028,7 +1127,7 @@ capi.wasm.scopedAllocPop(stack); } }else{ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); try{ rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); @@ -1042,7 +1141,7 @@ console.warn("Unsupported bind() argument type:",val); toss3("Unsupported bind() argument type: "+(typeof val)); } - if(rc) checkDbRc(stmt.db.pointer, rc); + if(rc) DB.checkRc(stmt.db.pointer, rc); return stmt; }; @@ -1059,6 +1158,7 @@ delete __stmtMap.get(this.db)[this.pointer]; capi.sqlite3_finalize(this.pointer); __ptrMap.delete(this); + delete this._mayGet; delete this.columnCount; delete this.parameterCount; delete this.db; @@ -1228,9 +1328,10 @@ return this; }, /** - Steps the statement one time. If the result indicates that - a row of data is available, true is returned. If no row of - data is available, false is returned. Throws on error. + Steps the statement one time. If the result indicates that a + row of data is available, a truthy value is returned. + If no row of data is available, a falsy + value is returned. Throws on error. */ step: function(){ affirmUnlocked(this, 'step()'); @@ -1242,8 +1343,51 @@ this._mayGet = false; console.warn("sqlite3_step() rc=",rc,"SQL =", capi.sqlite3_sql(this.pointer)); - checkDbRc(this.db.pointer, rc); - }; + DB.checkRc(this.db.pointer, rc); + } + }, + /** + Functions exactly like step() except that... + + 1) On success, it calls this.reset() and returns this object. + 2) On error, it throws and does not call reset(). + + This is intended to simplify constructs like: + + ``` + for(...) { + stmt.bind(...).stepReset(); + } + ``` + + Note that the reset() call makes it illegal to call this.get() + after the step. + */ + stepReset: function(){ + this.step(); + return this.reset(); + }, + /** + Functions like step() except that + it finalizes this statement immediately after stepping unless + the step cannot be performed because the statement is + locked. Throws on error, but any error other than the + statement-is-locked case will also trigger finalization of this + statement. + + On success, it returns true if the step indicated that a row of + data was available, else it returns false. + + This is intended to simplify use cases such as: + + ``` + aDb.prepare("insert in foo(a) values(?)").bind(123).stepFinalize(); + ``` + */ + stepFinalize: function(){ + const rc = this.step(); + this.finalize(); + return rc; }, /** Fetches the value from the given 0-based column index of @@ -1347,7 +1491,7 @@ default: toss3("Don't know how to translate", "type of result column #"+ndx+"."); } - abort("Not reached."); + toss3("Not reached."); }, /** Equivalent to get(ndx) but coerces the result to an integer. */ @@ -1434,5 +1578,20 @@ }, DB, Stmt - }/*SQLite3 object*/; -})(self); + }/*oo1 object*/; + + if( self.window===self && 0!==capi.sqlite3_vfs_find('kvvfs') ){ + /* In the main window thread, add a couple of convenience proxies + for localStorage and sessionStorage DBs... */ + let klass = sqlite3.oo1.LocalStorageDb = function(){ + dbCtorHelper.call(this, 'local', 'c', 'kvvfs'); + }; + klass.prototype = DB.prototype; + + klass = sqlite3.oo1.SessionStorageDb = function(){ + dbCtorHelper.call(this, 'session', 'c', 'kvvfs'); + }; + klass.prototype = DB.prototype; + } +}); + diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 4acab7770..693432b35 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -31,12 +31,13 @@ // FileSystemDirectoryHandle // FileSystemFileHandle // FileSystemFileHandle.prototype.createSyncAccessHandle -self.sqlite3.postInit.push(function(self, sqlite3){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const warn = console.warn.bind(console), error = console.error.bind(console); - if(!self.importScripts || !self.FileSystemFileHandle - || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ - warn("OPFS not found or its sync API is not available in this environment."); + if(!self.importScripts || !self.FileSystemFileHandle){ + //|| !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ + // ^^^ sync API is not required with WASMFS/OPFS backend. + warn("OPFS is not available in this environment."); return; }else if(!sqlite3.capi.wasm.bigIntEnabled){ error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 60ed61477..17dcd4228 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -78,25 +78,110 @@ */ /** - This global symbol is is only a temporary measure: the JS-side - post-processing will remove that object from the global scope when - setup is complete. We require it there temporarily in order to glue - disparate parts together during the loading of the API (which spans - several components). + sqlite3ApiBootstrap() is the only global symbol exposed by this + API. It is intended to be called one time at the end of the API + amalgamation process, passed configuration details for the current + environment, and then optionally be removed from the global object + using `delete self.sqlite3ApiBootstrap`. - This function requires a configuration object intended to abstract + This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so - that it can be used without any _direct_ dependency on Emscripten. - (That said, OO API #1 requires, as of this writing, Emscripten's - virtual filesystem API. Baby steps.) + that it can be used without any _direct_ dependency on + Emscripten. The config object is only honored the first time this + is called. Subsequent calls ignore the argument and return the same + (configured) object which gets initialized by the first call. + + The config object properties include: + + - `Module`[^1]: Emscripten-style module object. Currently only required + by certain test code and is _not_ part of the public interface. + (TODO: rename this to EmscriptenModule to be more explicit.) + + - `exports`[^1]: the "exports" object for the current WASM + environment. In an Emscripten build, this should be set to + `Module['asm']`. + + - `memory`[^1]: optional WebAssembly.Memory object, defaulting to + `exports.memory`. In Emscripten environments this should be set + to `Module.wasmMemory` if the build uses `-sIMPORT_MEMORY`, or be + left undefined/falsy to default to `exports.memory` when using + WASM-exported memory. + + - `bigIntEnabled`: true if BigInt support is enabled. Defaults to + true if self.BigInt64Array is available, else false. Some APIs + will throw exceptions if called without BigInt support, as BigInt + is required for marshalling C-side int64 into and out of JS. + + - `allocExportName`: the name of the function, in `exports`, of the + `malloc(3)`-compatible routine for the WASM environment. Defaults + to `"malloc"`. + + - `deallocExportName`: the name of the function, in `exports`, of + the `free(3)`-compatible routine for the WASM + environment. Defaults to `"free"`. + + - `persistentDirName`[^1]: if the environment supports persistent storage, this + directory names the "mount point" for that directory. It must be prefixed + by `/` and may currently contain only a single directory-name part. Using + the root directory name is not supported by any current persistent backend. + + + [^1] = This property may optionally be a function, in which case this + function re-assigns it to the value returned from that function, + enabling delayed evaluation. + */ -self.sqlite3ApiBootstrap = function(config){ - 'use strict'; +'use strict'; +self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (sqlite3ApiBootstrap.defaultConfig || self.sqlite3ApiConfig) +){ + if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ + console.warn("sqlite3ApiBootstrap() called multiple times.", + "Config and external initializers are ignored on calls after the first."); + return sqlite3ApiBootstrap.sqlite3; + } + apiConfig = apiConfig || {}; + const config = Object.create(null); + { + const configDefaults = { + Module: undefined/*needed for some test code, not part of the public API*/, + exports: undefined, + memory: undefined, + bigIntEnabled: !!self.BigInt64Array, + allocExportName: 'malloc', + deallocExportName: 'free', + persistentDirName: '/persistent' + }; + Object.keys(configDefaults).forEach(function(k){ + config[k] = Object.getOwnPropertyDescriptor(apiConfig, k) + ? apiConfig[k] : configDefaults[k]; + }); + // Copy over any properties apiConfig defines but configDefaults does not... + Object.keys(apiConfig).forEach(function(k){ + if(!Object.getOwnPropertyDescriptor(config, k)){ + config[k] = apiConfig[k]; + } + }); + } + + [ + // If any of these config options are functions, replace them with + // the result of calling that function... + 'Module', 'exports', 'memory', 'persistentDirName' + ].forEach((k)=>{ + if('function' === typeof config[k]){ + config[k] = config[k](); + } + }); /** Throws a new Error, the message of which is the concatenation all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; + if(config.persistentDirName && !/^\/[^/]+$/.test(config.persistentDirName)){ + toss("config.persistentDirName must be falsy or in the form '/dir-name'."); + } + /** Returns true if n is a 32-bit (signed) integer, else false. This is used for determining when we need to switch to @@ -143,7 +228,18 @@ self.sqlite3ApiBootstrap = function(config){ }; const utf8Decoder = new TextDecoder('utf-8'); - const typedArrayToString = (str)=>utf8Decoder.decode(str); + + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const typedArrayToString = function(arrayBuffer, begin, end){ + return utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; /** An Error subclass specifically for reporting Wasm-level malloc() @@ -173,36 +269,6 @@ self.sqlite3ApiBootstrap = function(config){ */ const capi = { /** - An Error subclass which is thrown by this object's alloc() method - on OOM. - */ - WasmAllocError: WasmAllocError, - /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). - - That said, very few cases in the API can result in - client-defined functions propagating exceptions via the C-style - API. Most notably, this applies ot User-defined SQL Functions - (UDFs) registered via sqlite3_create_function_v2(). For that - specific case it is recommended that all UDF creation be - funneled through a utility function and that a wrapper function - be added around the UDF which catches any exception and sets - the error state to OOM. (The overall complexity of registering - UDFs essentially requires a helper for doing so!) - */ - alloc: undefined/*installed later*/, - /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). - */ - dealloc: undefined/*installed later*/, - /** When using sqlite3_open_v2() it is important to keep the following in mind: @@ -365,6 +431,33 @@ self.sqlite3ApiBootstrap = function(config){ || toss("API config object requires a WebAssembly.Memory object", "in either config.exports.memory (exported)", "or config.memory (imported)."), + + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + That said, very few cases in the API can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies ot User-defined SQL Functions + (UDFs) registered via sqlite3_create_function_v2(). For that + specific case it is recommended that all UDF creation be + funneled through a utility function and that a wrapper function + be added around the UDF which catches any exception and sets + the error state to OOM. (The overall complexity of registering + UDFs essentially requires a helper for doing so!) + */ + alloc: undefined/*installed later*/, + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + */ + dealloc: undefined/*installed later*/ + /* Many more wasm-related APIs get installed later on. */ }/*wasm*/ }/*capi*/; @@ -387,7 +480,7 @@ self.sqlite3ApiBootstrap = function(config){ Int8Array types and will throw if srcTypedArray is of any other type. */ - capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + capi.wasm.allocFromTypedArray = function(srcTypedArray){ affirmBindableTypedArray(srcTypedArray); const pRet = this.alloc(srcTypedArray.byteLength || 1); this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); @@ -400,11 +493,13 @@ self.sqlite3ApiBootstrap = function(config){ const f = capi.wasm.exports[key]; if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); } + capi.wasm.alloc = function(n){ const m = this.exports[keyAlloc](n); if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); return m; }.bind(capi.wasm) + capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); /** @@ -576,18 +671,125 @@ self.sqlite3ApiBootstrap = function(config){ ["sqlite3_total_changes64", "i64", ["sqlite3*"]] ]; + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + capi.wasm. + */ + capi.wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_vfs_unlink", "int", "string"] + ]; + + /** State for sqlite3_web_persistent_dir(). */ + let __persistentDir; + /** + An experiment. Do not use. + + If the wasm environment has a persistent storage directory, + its path is returned by this function. If it does not then + it returns "" (noting that "" is a falsy value). + + The first time this is called, this function inspects the current + environment to determine whether persistence filesystem support + is available and, if it is, enables it (if needed). + + TODOs and caveats: + + - If persistent storage is available at the root of the virtual + filesystem, this interface cannot currently distinguish that + from the lack of persistence. That case cannot currently (with + WASMFS/OPFS) happen, but it is conceivably possible in future + environments or non-browser runtimes (none of which are yet + supported targets). + */ + capi.sqlite3_web_persistent_dir = function(){ + if(undefined !== __persistentDir) return __persistentDir; + // If we have no OPFS, there is no persistent dir + const pdir = config.persistentDirName; + if(!pdir + || !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return __persistentDir = ""; + } + try{ + if(pdir && 0===capi.wasm.xCallWrapped( + 'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir + )){ + /** OPFS does not support locking and will trigger errors if + we try to lock. We don't _really_ want to + _unconditionally_ install a non-locking sqlite3 VFS as the + default, but we do so here for simplicy's sake for the + time being. That said: locking is a no-op on all of the + current WASM storage, so this isn't (currently) as bad as + it may initially seem. */ + const pVfs = sqlite3.capi.sqlite3_vfs_find("unix-none"); + if(pVfs){ + capi.sqlite3_vfs_register(pVfs,1); + console.warn("Installed 'unix-none' as the default sqlite3 VFS."); + } + return __persistentDir = pdir; + }else{ + return __persistentDir = ""; + } + }catch(e){ + // sqlite3_wasm_init_opfs() is not available + return __persistentDir = ""; + } + }; + + /** + Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a + non-empty string and the given name has that string as its + prefix, else returns false. + */ + capi.sqlite3_web_filename_is_persistent = function(name){ + const p = capi.sqlite3_web_persistent_dir(); + return (p && name) ? name.startsWith(p) : false; + }; + + if(0===capi.wasm.exports.sqlite3_vfs_find(0)){ + /* Assume that sqlite3_initialize() has not yet been called. + This will be the case in an SQLITE_OS_KV build. */ + capi.wasm.exports.sqlite3_initialize(); + } + /* The remainder of the API will be set up in later steps. */ - return { + const sqlite3 = { + WasmAllocError: WasmAllocError, capi, - postInit: [ - /* some pieces of the API may install functions into this array, - and each such function will be called, passed (self,sqlite3), - at the very end of the API load/init process, where self is - the current global object and sqlite3 is the object returned - from sqlite3ApiBootstrap(). This array will be removed at the - end of the API setup process. */], - /** Config is needed downstream for gluing pieces together. It - will be removed at the end of the API setup process. */ config }; + sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3)); + delete sqlite3ApiBootstrap.initializers; + sqlite3ApiBootstrap.sqlite3 = sqlite3; + return sqlite3; }/*sqlite3ApiBootstrap()*/; +/** + self.sqlite3ApiBootstrap.initializers is an internal detail used by + the various pieces of the sqlite3 API's amalgamation process. It + must not be modified by client code except when plugging such code + into the amalgamation process. + + Each component of the amalgamation is expected to append a function + to this array. When sqlite3ApiBootstrap() is called for the first + time, each such function will be called (in their appended order) + and passed the sqlite3 namespace object, into which they can install + their features (noting that most will also require that certain + features alread have been installed). At the end of that process, + this array is deleted. +*/ +self.sqlite3ApiBootstrap.initializers = []; +/** + Client code may assign sqlite3ApiBootstrap.defaultConfig an + object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging in + an environment-suitable configuration without having to define a new + global-scope symbol. +*/ +self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +/** Placeholder: gets installed by the first call to + self.sqlite3ApiBootstrap(). */ +self.sqlite3ApiBootstrap.sqlite3 = undefined; diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js deleted file mode 100644 index 95b27b21e..000000000 --- a/ext/wasm/api/sqlite3-api-worker.js +++ /dev/null @@ -1,420 +0,0 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file implements a Worker-based wrapper around SQLite3 OO API - #1. - - In order to permit this API to be loaded in worker threads without - automatically registering onmessage handlers, initializing the - worker API requires calling initWorkerAPI(). If this function - is called from a non-worker thread then it throws an exception. - - When initialized, it installs message listeners to receive messages - from the main thread and then it posts a message in the form: - - ``` - {type:'sqlite3-api',data:'worker-ready'} - ``` - - This file requires that the core C-style sqlite3 API and OO API #1 - have been loaded and that self.sqlite3 contains both, - as documented for those APIs. -*/ -self.sqlite3.initWorkerAPI = function(){ - 'use strict'; - /** - UNDER CONSTRUCTION - - We need an API which can proxy the DB API via a Worker message - interface. The primary quirky factor in such an API is that we - cannot pass callback functions between the window thread and a - worker thread, so we have to receive all db results via - asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who wants - to juggle six database handles at once. Do we add that complexity - or tell such users to write their own code using the provided - lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we collect - all results and bundle them back in a single message? The former - is, generically speaking, more memory-efficient but the latter - far easier to implement in this environment. The latter is - untennable for large data sets. Despite a web page hypothetically - being a relatively limited environment, there will always be - those users who feel that they should/need to be able to work - with multi-hundred-meg (or larger) blobs, and passing around - arrays of those may quickly exhaust the JS engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB.pointer (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB.pointer) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. - - - Revisit how virtual files are managed. We currently delete DBs - from the virtual filesystem when we close them, for the sake of - saving memory (the VFS lives in RAM). Supporting multiple DBs may - require that we give up that habit. Similarly, fully supporting - ATTACH, where a user can upload multiple DBs and ATTACH them, - also requires the that we manage the VFS entries better. - */ - const toss = (...args)=>{throw new Error(args.join(' '))}; - if('function' !== typeof importScripts){ - toss("Cannot initalize the sqlite3 worker API in the main thread."); - } - const self = this.self; - const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); - const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); - const DB = SQLite3.DB; - - /** - Returns the app-wide unique ID for the given db, creating one if - needed. - */ - const getDbId = function(db){ - let id = wState.idMap.get(db); - if(id) return id; - id = 'db#'+(++wState.idSeq)+'@'+db.pointer; - /** ^^^ can't simply use db.pointer b/c closing/opening may re-use - the same address, which could map pending messages to a wrong - instance. */ - wState.idMap.set(db, id); - return id; - }; - - /** - Helper for managing Worker-level state. - */ - const wState = { - defaultDb: undefined, - idSeq: 0, - idMap: new WeakMap, - open: function(arg){ - // TODO: if arg is a filename, look for a db in this.dbs with the - // same filename and close/reopen it (or just pass it back as is?). - if(!arg && this.defaultDb) return this.defaultDb; - //???if(this.defaultDb) this.defaultDb.close(); - let db; - db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); - this.dbs[getDbId(db)] = db; - if(!this.defaultDb) this.defaultDb = db; - return db; - }, - close: function(db,alsoUnlink){ - if(db){ - delete this.dbs[getDbId(db)]; - db.close(alsoUnlink); - if(db===this.defaultDb) this.defaultDb = undefined; - } - }, - post: function(type,data,xferList){ - if(xferList){ - self.postMessage({type, data},xferList); - xferList.length = 0; - }else{ - self.postMessage({type, data}); - } - }, - /** Map of DB IDs to DBs. */ - dbs: Object.create(null), - getDb: function(id,require=true){ - return this.dbs[id] - || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); - } - }; - - /** Throws if the given db is falsy or not opened. */ - const affirmDbOpen = function(db = wState.defaultDb){ - return (db && db.pointer) ? db : toss("DB is not opened."); - }; - - /** Extract dbId from the given message payload. */ - const getMsgDb = function(msgData,affirmExists=true){ - const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; - return affirmExists ? affirmDbOpen(db) : db; - }; - - const getDefaultDbId = function(){ - return wState.defaultDb && getDbId(wState.defaultDb); - }; - - /** - A level of "organizational abstraction" for the Worker - API. Each method in this object must map directly to a Worker - message type key. The onmessage() dispatcher attempts to - dispatch all inbound messages to a method of this object, - passing it the event.data part of the inbound event object. All - methods must return a plain Object containing any response - state, which the dispatcher may amend. All methods must throw - on error. - */ - const wMsgHandler = { - xfer: [/*Temp holder for "transferable" postMessage() state.*/], - /** - Proxy for DB.exec() which expects a single argument of type - string (SQL to execute) or an options object in the form - expected by exec(). The notable differences from exec() - include: - - - The default value for options.rowMode is 'array' because - the normal default cannot cross the window/Worker boundary. - - - A function-type options.callback property cannot cross - the window/Worker boundary, so is not useful here. If - options.callback is a string then it is assumed to be a - message type key, in which case a callback function will be - applied which posts each row result via: - - postMessage({type: thatKeyType, data: theRow}) - - And, at the end of the result set (whether or not any - result rows were produced), it will post an identical - message with data:null to alert the caller than the result - set is completed. - - The callback proxy must not recurse into this interface, or - results are undefined. (It hypothetically cannot recurse - because an exec() call will be tying up the Worker thread, - causing any recursion attempt to wait until the first - exec() is completed.) - - The response is the input options object (or a synthesized - one if passed only a string), noting that - options.resultRows and options.columnNames may be populated - by the call to exec(). - - This opens/creates the Worker's db if needed. - */ - exec: function(ev){ - const opt = ( - 'string'===typeof ev.data - ) ? {sql: ev.data} : (ev.data || Object.create(null)); - if(undefined===opt.rowMode){ - /* Since the default rowMode of 'stmt' is not useful - for the Worker interface, we'll default to - something else. */ - opt.rowMode = 'array'; - }else if('stmt'===opt.rowMode){ - toss("Invalid rowMode for exec(): stmt mode", - "does not work in the Worker API."); - } - const db = getMsgDb(ev); - if(opt.callback || Array.isArray(opt.resultRows)){ - // Part of a copy-avoidance optimization for blobs - db._blobXfer = this.xfer; - } - const callbackMsgType = opt.callback; - if('string' === typeof callbackMsgType){ - /* Treat this as a worker message type and post each - row as a message of that type. */ - const that = this; - opt.callback = - (row)=>wState.post(callbackMsgType,row,this.xfer); - } - try { - db.exec(opt); - if(opt.callback instanceof Function){ - opt.callback = callbackMsgType; - wState.post(callbackMsgType, null); - } - }/*catch(e){ - console.warn("Worker is propagating:",e);throw e; - }*/finally{ - delete db._blobXfer; - if(opt.callback){ - opt.callback = callbackMsgType; - } - } - return opt; - }/*exec()*/, - /** - TO(re)DO, once we can abstract away access to the - JS environment's virtual filesystem. Currently this - always throws. - - Response is (should be) an object: - - { - buffer: Uint8Array (db file contents), - filename: the current db filename, - mimetype: 'application/x-sqlite3' - } - - TODO is to determine how/whether this feature can support - exports of ":memory:" and "" (temp file) DBs. The latter is - ostensibly easy because the file is (potentially) on disk, but - the former does not have a structure which maps directly to a - db file image. - */ - export: function(ev){ - toss("export() requires reimplementing for portability reasons."); - /**const db = getMsgDb(ev); - const response = { - buffer: db.exportBinaryImage(), - filename: db.filename, - mimetype: 'application/x-sqlite3' - }; - this.xfer.push(response.buffer.buffer); - return response;**/ - }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input), - - id: an opaque ID value intended for future distinction - between multiple db handles. Messages including a specific - ID will use the DB for that ID. - - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.simulateError){ - toss("Throwing because of open.simulateError flag."); - } - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return { - filename: db.filename, - dbId: getDbId(db) - }; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - { - filename: db filename _if_ the db is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const db = getMsgDb(ev,false); - const response = { - filename: db && db.filename - }; - if(db){ - wState.close(db, !!((ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data)); - } - return response; - }, - toss: function(ev){ - toss("Testing worker exception"); - } - }/*wMsgHandler*/; - - /** - UNDER CONSTRUCTION! - - A subset of the DB API is accessible via Worker messages in the - form: - - { type: apiCommand, - dbId: optional DB ID value (else uses a default db handle) - data: apiArguments - } - - As a rule, these commands respond with a postMessage() of their - own in the same form, but will, if needed, transform the `data` - member to an object and may add state to it. The responses - always have an object-format `data` part. If the inbound `data` - is an object which has a `messageId` property, that property is - always mirrored in the result object, for use in client-side - dispatching of these asynchronous results. Exceptions thrown - during processing result in an `error`-type event with a - payload in the form: - - { - message: error string, - errorClass: class name of the error type, - dbId: DB handle ID, - input: ev.data, - [messageId: if set in the inbound message] - } - - The individual APIs are documented in the wMsgHandler object. - */ - self.onmessage = function(ev){ - ev = ev.data; - let response, dbId = ev.dbId, evType = ev.type; - const arrivalTime = performance.now(); - try { - if(wMsgHandler.hasOwnProperty(evType) && - wMsgHandler[evType] instanceof Function){ - response = wMsgHandler[evType](ev); - }else{ - toss("Unknown db worker message type:",ev.type); - } - }catch(err){ - evType = 'error'; - response = { - message: err.message, - errorClass: err.name, - input: ev - }; - if(err.stack){ - response.stack = ('string'===typeof err.stack) - ? err.stack.split('\n') : err.stack; - } - if(0) console.warn("Worker is propagating an exception to main thread.", - "Reporting it _here_ for the stack trace:",err,response); - } - if(!response.messageId && ev.data - && 'object'===typeof ev.data && ev.data.messageId){ - response.messageId = ev.data.messageId; - } - if(!dbId){ - dbId = response.dbId/*from 'open' cmd*/ - || getDefaultDbId(); - } - if(!response.dbId) response.dbId = dbId; - // Timing info is primarily for use in testing this API. It's not part of - // the public API. arrivalTime = when the worker got the message. - response.workerReceivedTime = arrivalTime; - response.workerRespondTime = performance.now(); - response.departureTime = ev.departureTime; - wState.post(evType, response, wMsgHandler.xfer); - }; - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); -}.bind({self, sqlite3: self.sqlite3}); diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js new file mode 100644 index 000000000..00359413b --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -0,0 +1,621 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + sqlite3.initWorker1API() implements a Worker-based wrapper around + SQLite3 OO API #1, colloquially known as "Worker API #1". + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorker1API(). If this function is + called from a non-worker thread then it throws an exception. It + must only be called once per Worker. + + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: + + ``` + {type:'sqlite3-api', result:'worker1-ready'} + ``` + + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. + + Note that the worker-based interface can be slightly quirky because + of its async nature. In particular, any number of messages may be posted + to the worker before it starts handling any of them. If, e.g., an + "open" operation fails, any subsequent messages will fail. The + Promise-based wrapper for this API (`sqlite3-worker1-promiser.js`) + is more comfortable to use in that regard. + + The documentation for the input and output worker messages for + this API follows... + + ==================================================================== + Common message format... + + Each message posted to the worker has an operation-independent + envelope and operation-dependent arguments: + + ``` + { + type: string, // one of: 'open', 'close', 'exec', 'config-get' + + messageId: OPTIONAL arbitrary value. The worker will copy it as-is + into response messages to assist in client-side dispatching. + + dbId: a db identifier string (returned by 'open') which tells the + operation which database instance to work on. If not provided, the + first-opened db is used. This is an "opaque" value, with no + inherently useful syntax or information. Its value is subject to + change with any given build of this API and cannot be used as a + basis for anything useful beyond its one intended purpose. + + args: ...operation-dependent arguments... + + // the framework may add other properties for testing or debugging + // purposes. + + } + ``` + + Response messages, posted back to the main thread, look like: + + ``` + { + type: string. Same as above except for error responses, which have the type + 'error', + + messageId: same value, if any, provided by the inbound message + + dbId: the id of the db which was operated on, if any, as returned + by the corresponding 'open' operation. + + result: ...operation-dependent result... + + } + ``` + + ==================================================================== + Error responses + + Errors are reported messages in an operation-independent format: + + ``` + { + type: 'error', + + messageId: ...as above..., + + dbId: ...as above... + + result: { + + operation: type of the triggering operation: 'open', 'close', ... + + message: ...error message text... + + errorClass: string. The ErrorClass.name property from the thrown exception. + + input: the message object which triggered the error. + + stack: _if available_, a stack trace array. + + } + + } + ``` + + + ==================================================================== + "config-get" + + This operation fetches the serializable parts of the sqlite3 API + configuration. + + Message format: + + ``` + { + type: "config-get", + messageId: ...as above..., + args: currently ignored and may be elided. + } + ``` + + Response: + + ``` + { + type: 'config', + messageId: ...as above..., + result: { + + persistentDirName: path prefix, if any, of persistent storage. + An empty string denotes that no persistent storage is available. + + bigIntEnabled: bool. True if BigInt support is enabled. + + persistenceEnabled: true if persistent storage is enabled in the + current environment. Only files stored under persistentDirName + will persist, however. + + } + } + ``` + + + ==================================================================== + "open" a database + + Message format: + + ``` + { + type: "open", + messageId: ...as above..., + args:{ + + filename [=":memory:" or "" (unspecified)]: the db filename. + See the sqlite3.oo1.DB constructor for peculiarities and transformations, + + persistent [=false]: if true and filename is not one of ("", + ":memory:"), prepend sqlite3.capi.sqlite3_web_persistent_dir() + to the given filename so that it is stored in persistent storage + _if_ the environment supports it. If persistent storage is not + supported, the filename is used as-is. + + } + } + ``` + + Response: + + ``` + { + type: 'open', + messageId: ...as above..., + result: { + filename: db filename, possibly differing from the input. + + dbId: an opaque ID value which must be passed in the message + envelope to other calls in this API to tell them which db to + use. If it is not provided to future calls, they will default to + operating on the first-opened db. This property is, for API + consistency's sake, also part of the contaning message envelope. + Only the `open` operation includes it in the `result` property. + + persistent: true if the given filename resides in the + known-persistent storage, else false. This determination is + independent of the `persistent` input argument. + } + } + ``` + + ==================================================================== + "close" a database + + Message format: + + ``` + { + type: "close", + messageId: ...as above... + dbId: ...as above... + args: OPTIONAL: { + + unlink: if truthy, the associated db will be unlinked (removed) + from the virtual filesystems. Failure to unlink is silently + ignored. + + } + } + ``` + + If the dbId does not refer to an opened ID, this is a no-op. The + inability to close a db (because it's not opened) or delete its + file does not trigger an error. + + Response: + + ``` + { + type: 'close', + messageId: ...as above..., + result: { + + filename: filename of closed db, or undefined if no db was closed + + } + } + ``` + + ==================================================================== + "exec" SQL + + All SQL execution is processed through the exec operation. It offers + most of the features of the oo1.DB.exec() method, with a few limitations + imposed by the state having to cross thread boundaries. + + Message format: + + ``` + { + type: "exec", + messageId: ...as above... + dbId: ...as above... + args: string (SQL) or {... see below ...} + } + ``` + + Response: + + ``` + { + type: 'exec', + messageId: ...as above..., + dbId: ...as above... + result: { + input arguments, possibly modified. See below. + } + } + ``` + + The arguments are in the same form accepted by oo1.DB.exec(), with + the exceptions noted below. + + A function-type args.callback property cannot cross + the window/Worker boundary, so is not useful here. If + args.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, + rowNumber: 1-based-#, + row: theRow, + columnNames: anArray + }) + + And, at the end of the result set (whether or not any result rows + were produced), it will post an identical message with + (row=undefined, rowNumber=null) to alert the caller than the result + set is completed. Note that a row value of `null` is a legal row + result for certain arg.rowMode values. + + (Design note: we don't use (row=undefined, rowNumber=undefined) to + indicate end-of-results because fetching those would be + indistinguishable from fetching from an empty object unless the + client used hasOwnProperty() (or similar) to distinguish "missing + property" from "property with the undefined value". Similarly, + `null` is a legal value for `row` in some case , whereas the db + layer won't emit a result value of `undefined`.) + + The callback proxy must not recurse into this interface. An exec() + call will type up the Worker thread, causing any recursion attempt + to wait until the first exec() is completed. + + The response is the input options object (or a synthesized one if + passed only a string), noting that options.resultRows and + options.columnNames may be populated by the call to db.exec(). + +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +sqlite3.initWorker1API = function(){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("Cannot initalize the sqlite3 worker API in the main thread."); + } + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); + const DB = SQLite3.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Internal helper for managing Worker-level state. + */ + const wState = { + /** First-opened db is the default for future operations when no + dbId is provided by the client. */ + defaultDb: undefined, + /** Sequence number of dbId generation. */ + idSeq: 0, + /** Map of DB instances to dbId. */ + idMap: new WeakMap, + /** Temp holder for "transferable" postMessage() state. */ + xfer: [], + open: function(opt){ + const db = new DB(opt.filename); + this.dbs[getDbId(db)] = db; + if(!this.defaultDb) this.defaultDb = db; + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + const filename = db.fileName(); + db.close(); + if(db===this.defaultDb) this.defaultDb = undefined; + if(alsoUnlink && filename){ + sqlite3.capi.wasm.sqlite3_wasm_vfs_unlink(filename); + } + } + }, + /** + Posts the given worker message value. If xferList is provided, + it must be an array, in which case a copy of it passed as + postMessage()'s second argument and xferList.length is set to + 0. + */ + post: function(msg,xferList){ + if(xferList && xferList.length){ + self.postMessage( msg, Array.from(xferList) ); + xferList.length = 0; + }else{ + self.postMessage(msg); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + /** Fetch the DB for the given id. Throw if require=true and the + id is not valid, else return the db or undefined. */ + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened. */ + const affirmDbOpen = function(db = wState.defaultDb){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.defaultDb && getDbId(wState.defaultDb); + }; + + /** + A level of "organizational abstraction" for the Worker + API. Each method in this object must map directly to a Worker + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any result + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + open: function(ev){ + const oargs = Object.create(null), args = (ev.args || Object.create(null)); + if(args.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + const rc = Object.create(null); + const pDir = sqlite3.capi.sqlite3_web_persistent_dir(); + if(!args.filename || ':memory:'===args.filename){ + oargs.filename = args.filename || ''; + }else if(pDir){ + oargs.filename = pDir + ('/'===args.filename[0] ? args.filename : ('/'+args.filename)); + }else{ + oargs.filename = args.filename; + } + const db = wState.open(oargs); + rc.filename = db.filename; + rc.persistent = !!pDir && db.filename.startsWith(pDir); + rc.dbId = getDbId(db); + return rc; + }, + + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + wState.close(db, ((ev.args && 'object'===typeof ev.args) + ? !!ev.args.unlink : false)); + } + return response; + }, + + exec: function(ev){ + const rc = ( + 'string'===typeof ev.args + ) ? {sql: ev.args} : (ev.args || Object.create(null)); + if('stmt'===rc.rowMode){ + toss("Invalid rowMode for 'exec': stmt mode", + "does not work in the Worker API."); + }else if(!rc.sql){ + toss("'exec' requires input SQL."); + } + const db = getMsgDb(ev); + if(rc.callback || Array.isArray(rc.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = wState.xfer; + } + const theCallback = rc.callback; + let rowNumber = 0; + const hadColNames = !!rc.columnNames; + if('string' === typeof theCallback){ + if(!hadColNames) rc.columnNames = []; + /* Treat this as a worker message type and post each + row as a message of that type. */ + rc.callback = function(row,stmt){ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: ++rowNumber, + row: row + }, wState.xfer); + } + } + try { + db.exec(rc); + if(rc.callback instanceof Function){ + rc.callback = theCallback; + /* Post a sentinel message to tell the client that the end + of the result set has been reached (possibly with zero + rows). */ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: null /*null to distinguish from "property not set"*/, + row: undefined /*undefined because null is a legal row value + for some rowType values, but undefined is not*/ + }); + } + }finally{ + delete db._blobXfer; + if(rc.callback) rc.callback = theCallback; + } + return rc; + }/*exec()*/, + + 'config-get': function(){ + const rc = Object.create(null), src = sqlite3.config; + [ + 'persistentDirName', 'bigIntEnabled' + ].forEach(function(k){ + if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; + }); + rc.persistenceEnabled = !!sqlite3.capi.sqlite3_web_persistent_dir(); + return rc; + }, + + /** + TO(RE)DO, once we can abstract away access to the + JS environment's virtual filesystem. Currently this + always throws. + + Response is (should be) an object: + + { + buffer: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + + TODO is to determine how/whether this feature can support + exports of ":memory:" and "" (temp file) DBs. The latter is + ostensibly easy because the file is (potentially) on disk, but + the former does not have a structure which maps directly to a + db file image. We can VACUUM INTO a :memory:/temp db into a + file for that purpose, though. + */ + export: function(ev){ + toss("export() requires reimplementing for portability reasons."); + /** + We need to reimplement this to use the Emscripten FS + interface. That part used to be in the OO#1 API but that + dependency was removed from that level of the API. + */ + /**const db = getMsgDb(ev); + const response = { + buffer: db.exportBinaryImage(), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + wState.xfer.push(response.buffer.buffer); + return response;**/ + }/*export()*/, + + toss: function(ev){ + toss("Testing worker exception"); + } + }/*wMsgHandler*/; + + self.onmessage = function(ev){ + ev = ev.data; + let result, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + result = wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + result = { + operation: ev.type, + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + result.stack = ('string'===typeof err.stack) + ? err.stack.split(/\n\s*/) : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,result); + } + if(!dbId){ + dbId = result.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + wState.post({ + type: evType, + dbId: dbId, + messageId: ev.messageId, + workerReceivedTime: arrivalTime, + workerRespondTime: performance.now(), + departureTime: ev.departureTime, + // TODO: move the timing bits into... + //timing:{ + // departure: ev.departureTime, + // workerReceived: arrivalTime, + // workerResponse: performance.now(); + //}, + result: result + }, wState.xfer); + }; + self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); +}.bind({self, sqlite3}); +}); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 6a81da3e5..2a505f19a 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1,6 +1,40 @@ +/* +** This file requires access to sqlite3.c static state in order to +** implement certain WASM-specific features. Unlike the rest of +** sqlite3.c, this file requires compiling with -std=c99 (or +** equivalent, or a later C version) because it makes use of features +** not available in C89. +*/ #include "sqlite3.c" /* +** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not +** Emscripten-specific. It explicitly includes marked functions for +** export into the target wasm file without requiring explicit listing +** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list +** (or equivalent in other build platforms). Any function with neither +** this attribute nor which is listed as an explicit export will not +** be exported from the wasm file (but may still be used internally +** within the wasm file). +** +** The functions in this file (sqlite3-wasm.c) which require exporting +** are marked with this flag. They may also be added to any explicit +** build-time export list but need not be. All of these APIs are +** intended for use only within the project's own JS/WASM code, and +** not by client code, so an argument can be made for reducing their +** visibility by not including them in any build-time export lists. +** +** 2022-09-11: it's not yet _proven_ that this approach works in +** non-Emscripten builds. If not, such builds will need to export +** those using the --export=... wasm-ld flag (or equivalent). As of +** this writing we are tied to Emscripten for various reasons +** and cannot test the library with other build environments. +*/ +#define WASM_KEEP __attribute__((used,visibility("default"))) +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) + +/* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** @@ -14,8 +48,8 @@ ** ** Returns err_code. */ -int sqlite3_wasm_db_error(sqlite3*db, int err_code, - const char *zMsg){ +WASM_KEEP +int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ if(0!=zMsg){ const int nMsg = sqlite3Strlen30(zMsg); sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); @@ -40,6 +74,7 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, ** buffer is not large enough for the generated JSON. In debug builds ** that will trigger an assert(). */ +WASM_KEEP const char * sqlite3_wasm_enum_json(void){ static char strBuf[1024 * 8] = {0} /* where the JSON goes */; int n = 0, childCount = 0, structCount = 0 @@ -411,3 +446,82 @@ const char * sqlite3_wasm_enum_json(void){ #undef outf #undef lenCheck } + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function invokes the xDelete method of the default VFS, +** passing on the given filename. If zName is NULL, no default VFS is +** found, or it has no xDelete method, SQLITE_MISUSE is returned, else +** the result of the xDelete() call is returned. +*/ +WASM_KEEP +int sqlite3_wasm_vfs_unlink(const char * zName){ + int rc = SQLITE_MISUSE /* ??? */; + sqlite3_vfs * const pVfs = sqlite3_vfs_find(0); + if( zName && pVfs && pVfs->xDelete ){ + rc = pVfs->xDelete(pVfs, zName, 1); + } + return rc; +} + +#if defined(__EMSCRIPTEN__) && defined(SQLITE_WASM_OPFS) +#include <emscripten/wasmfs.h> +#include <emscripten/console.h> + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings, specifically +** only when building with Emscripten's WASMFS support. +** +** This function should only be called if the JS side detects the +** existence of the Origin-Private FileSystem (OPFS) APIs in the +** client. The first time it is called, this function instantiates a +** WASMFS backend impl for OPFS. On success, subsequent calls are +** no-ops. +** +** This function may be passed a "mount point" name, which must have a +** leading "/" and is currently restricted to a single path component, +** e.g. "/foo" is legal but "/foo/" and "/foo/bar" are not. If it is +** NULL or empty, it defaults to "/persistent". +** +** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend +** object fails, SQLITE_IOERR if mkdir() of the zMountPoint dir in +** the virtual FS fails. In builds compiled without SQLITE_WASM_OPFS +** defined, SQLITE_NOTFOUND is returned without side effects. +*/ +WASM_KEEP +int sqlite3_wasm_init_opfs(const char *zMountPoint){ + static backend_t pOpfs = 0; + if( !zMountPoint || !*zMountPoint ) zMountPoint = "/persistent"; + if( !pOpfs ){ + pOpfs = wasmfs_create_opfs_backend(); + if( pOpfs ){ + emscripten_console_log("Created WASMFS OPFS backend."); + } + } + /** It's not enough to instantiate the backend. We have to create a + mountpoint in the VFS and attach the backend to it. */ + if( pOpfs && 0!=access(zMountPoint, F_OK) ){ + /* mkdir() simply hangs when called from fiddle app. Cause is + not yet determined but the hypothesis is an init-order + issue. */ + /* Note that this check and is not robust but it will + hypothetically suffice for the transient wasm-based virtual + filesystem we're currently running in. */ + const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs); + emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc); + if(rc) return SQLITE_IOERR; + } + return pOpfs ? 0 : SQLITE_NOMEM; +} +#else +WASM_KEEP +int sqlite3_wasm_init_opfs(void){ + return SQLITE_NOTFOUND; +} +#endif /* __EMSCRIPTEN__ && SQLITE_WASM_OPFS */ + + +#undef WASM_KEEP diff --git a/ext/wasm/api/sqlite3-worker.js b/ext/wasm/api/sqlite3-worker.js deleted file mode 100644 index 48797de8a..000000000 --- a/ext/wasm/api/sqlite3-worker.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - 2022-05-23 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This is a JS Worker file for the main sqlite3 api. It loads - sqlite3.js, initializes the module, and postMessage()'s a message - after the module is initialized: - - {type: 'sqlite3-api', data: 'worker-ready'} - - This seemingly superfluous level of indirection is necessary when - loading sqlite3.js via a Worker. Instantiating a worker with new - Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to - initialize the module due to a timing/order-of-operations conflict - (and that symbol is not exported in a way that a Worker loading it - that way can see it). Thus JS code wanting to load the sqlite3 - Worker-specific API needs to pass _this_ file (or equivalent) to the - Worker constructor and then listen for an event in the form shown - above in order to know when the module has completed initialization. -*/ -"use strict"; -importScripts('sqlite3.js'); -sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorkerAPI()); |