diff options
| -rw-r--r-- | ext/wasm/api/sqlite3-api-glue.js | 2 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-api-prologue.js | 47 | ||||
| -rw-r--r-- | ext/wasm/api/sqlite3-wasm.c | 78 | ||||
| -rw-r--r-- | ext/wasm/fiddle.make | 5 | ||||
| -rw-r--r-- | ext/wasm/fiddle/fiddle-worker.js | 98 | ||||
| -rw-r--r-- | manifest | 22 | ||||
| -rw-r--r-- | manifest.uuid | 2 | ||||
| -rw-r--r-- | src/os_kv.c | 12 |
8 files changed, 151 insertions, 115 deletions
diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 347c03bc4..ab9424aca 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -291,7 +291,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ For the given integer, returns the SQLITE_xxx result code as a string, or undefined if no such mapping is found. */ - capi.sqlite3_wasm_rc_str = (rc)=>__rcMap[rc]; + capi.sqlite3_web_rc_str = (rc)=>__rcMap[rc]; /* Bind all registered C-side structs... */ for(const s of wasm.ctype.structs){ capi[s.name] = sqlite3.StructBinder(s); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a6e055ff4..121b3d4ff 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -870,6 +870,51 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return rc; }; + /** + Serializes the given `sqlite3*` pointer to a Uint8Array, as per + sqlite3_serialize(). On success it returns a Uint8Array. On + error it throws with a description of the problem. + */ + capi.sqlite3_web_db_export = function(pDb){ + if(!pDb) toss('Invalid sqlite3* argument.'); + const wasm = capi.wasm; + if(!wasm.bigIntEnabled) toss('BigInt64 support is not enabled.'); + const scope = wasm.scopedAllocPush(); + let pOut; + try{ + const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof); + const ppOut = pSize + 8; + /** + Maintenance reminder, since this cost a full hour of grief + and confusion: if the order of pSize/ppOut are reversed in + that memory block, fetching the value of pSize after the + export reads a garbage size because it's not on an 8-byte + memory boundary! + */ + wasm.setPtrValue(ppOut, 0); + wasm.setMemValue(pSize, 0, 'i64'); + let rc = wasm.exports.sqlite3_wasm_db_serialize( + pDb, ppOut, pSize, 0 + ); + if(rc){ + toss("Database serialization failed with code", + sqlite3.capi.sqlite3_web_rc_str(rc)); + } + const pOut = wasm.getPtrValue(ppOut); + const nOut = wasm.getMemValue(pSize, 'i64'); + rc = nOut + ? wasm.heap8u().slice(pOut, pOut + Number(nOut)) + : new Uint8Array(); + return rc; + }catch(e){ + console.error('internal error?',e); + throw w; + }finally{ + if(pOut) wasm.exports.sqlite3_free(pOut); + wasm.scopedAllocPop(scope); + } + }; + if( capi.util.isMainWindow() ){ /* Features specific to the main window thread... */ @@ -945,7 +990,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } } }); - return sz * 2 /* because JS uses UC16 encoding */; + return sz * 2 /* because JS uses 2-byte char encoding */; }; }/* main-window-only bits */ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index da8f5c32e..a66b51a8f 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -406,6 +406,7 @@ const char * sqlite3_wasm_enum_json(void){ } _DefGroup; DefGroup(serialize){ + DefInt(SQLITE_SERIALIZE_NOCOPY); DefInt(SQLITE_DESERIALIZE_FREEONCLOSE); DefInt(SQLITE_DESERIALIZE_READONLY); DefInt(SQLITE_DESERIALIZE_RESIZEABLE); @@ -567,6 +568,83 @@ int sqlite3_wasm_vfs_unlink(const char * zName){ return rc; } +/* +** Uses the current database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +** +** This implementation appears to work fine, but +** sqlite3_wasm_db_serialize() is arguably the better way to achieve +** this. +*/ +WASM_KEEP +int sqlite3_wasm_db_export_chunked( sqlite3* pDb, + int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = pDb + ? sqlite3_file_control(pDb, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size + ** with zero-padding when exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPos<nSize; nPos += nBuf ){ + rc = pFile->pMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } + return rc; +} + +/* +** A proxy for sqlite3_serialize() which serializes the "main" schema +** of pDb, placing the serialized output in pOut and nOut. nOut may be +** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If +** allocation of the serialized copy fails, SQLITE_NOMEM is returned. +** On success, 0 is returned and `*pOut` will contain a pointer to the +** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the +** database has no contiguous memory representation, in which case +** `*pOut` will be NULL but 0 will be returned. +** +** If `*pOut` is not NULL, the caller is responsible for passing it to +** sqlite3_free() to free it. +*/ +WASM_KEEP +int sqlite3_wasm_db_serialize( sqlite3* pDb, unsigned char **pOut, sqlite3_int64 * nOut, + unsigned int mFlags ){ + unsigned char * z; + if( !pDb || !pOut ) return SQLITE_MISUSE; + if(nOut) *nOut = 0; + z = sqlite3_serialize(pDb, "main", nOut, mFlags); + if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ + *pOut = z; + return 0; + }else{ + return SQLITE_NOMEM; + } +} + + #if defined(__EMSCRIPTEN__) #include <emscripten/console.h> #if defined(SQLITE_WASM_WASMFS) diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 6ccf8e029..585add427 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -37,7 +37,6 @@ fiddle.emcc-flags = \ $(sqlite3.js.flags.--post-js) \ -sEXPORTED_RUNTIME_METHODS=@$(dir.wasm)/EXPORTED_RUNTIME_METHODS.fiddle \ -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.fiddle \ - --post-js=$(post-js.js) \ $(SQLITE_OPT) $(SHELL_OPT) \ -DSQLITE_SHELL_FIDDLE # -D_POSIX_C_SOURCE is needed for strdup() with emcc @@ -60,9 +59,9 @@ $(dir.fiddle)/$(SOAP.js): $(SOAP.js) $(eval $(call call-make-pre-js,fiddle-module)) $(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \ EXPORTED_FUNCTIONS.fiddle EXPORTED_RUNTIME_METHODS.fiddle \ - $(fiddle.cs) $(pre-post-fiddle.deps) $(dir.fiddle)/$(SOAP.js) + $(fiddle.cs) $(pre-post-fiddle-module.deps) $(dir.fiddle)/$(SOAP.js) $(emcc.bin) -o $@ $(fiddle.emcc-flags) \ - $(pre-post-common.flags) $(pre-post-fiddle.flags) \ + $(pre-post-common.flags) $(pre-post-fiddle-module.flags) \ $(fiddle.cs) $(maybe-wasm-strip) $(fiddle-module.wasm) gzip < $@ > $@.gz diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index 6a0cf53b7..ef2e3fcb5 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -227,94 +227,6 @@ f._(); } }; - - /** - Exports the shell's current db file in such a way that it can - export DBs hosted in the Emscripten-supplied FS or in native OPFS - (and, hypothetically, kvvfs). Throws on error. On success returns - a Blob containing the whole db contents. - - Bug/to investigate: xFileSize() is returning garbage for the - default VFS but works in OPFS. Thus for exporting that impl we'll - use the fiddleModule.FS API for the time being. The equivalent - native impl, fiddle_export_db(), works okay with both VFSes, so - the bug is apparently in (or via) this code. - */ - const brokenExportDbFileToBlob = function(){ - const capi = sqlite3.capi, wasm = capi.wasm; - const pDb = Sqlite3Shell.dbHandle(); - if(!pDb) toss("No db is opened."); - const scope = wasm.scopedAllocPush(); - try{ - const ppFile = wasm.scopedAlloc(12/*sizeof(i32 + i64)*/); - const pFileSize = ppFile + 4; - wasm.setMemValue(ppFile, 0, '*'); - let rc = capi.sqlite3_file_control( - pDb, "main", capi.SQLITE_FCNTL_FILE_POINTER, ppFile - ); - if(rc) toss("Cannot get sqlite3_file handle."); - const jFile = new capi.sqlite3_file(wasm.getPtrValue(ppFile)); - const jIom = new capi.sqlite3_io_methods(jFile.$pMethods); - const xFileSize = wasm.functionEntry(jIom.$xFileSize); - const xRead = wasm.functionEntry(jIom.$xRead); - wasm.setMemValue(pFileSize, 0, 'i64'); - //stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize); - rc = xFileSize( jFile.pointer, pFileSize ); - if(rc) toss("Cannot get db file size."); - //stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize); - const nFileSize = Number( wasm.getMemValue(pFileSize,'i64') ); - if(nFileSize <= 0n || nFileSize>=Number.MAX_SAFE_INTEGER){ - toss("Unexpected DB size:",nFileSize); - } - //stderr("nFileSize =",nFileSize,"pFileSize =",pFileSize); - const nIobuf = 1024 * 4; - const iobuf = wasm.scopedAlloc(nIobuf); - let nPos = 0; - const blobList = []; - const heap = wasm.heap8u(); - for( ; nPos < nFileSize; nPos += nIobuf ){ - rc = xRead(jFile.pointer, iobuf, nIobuf, BigInt(nPos)); - if(rc){ - if(capi.SQLITE_IOERR_SHORT_READ === rc){ - //stderr('rc =',rc,'nPos =',nPos,'nIobuf =',nIobuf,'nFileSize =',nFileSize); - rc = ((nPos + nIobuf) < nFileSize) ? rc : 0/*assume EOF*/; - } - if(rc) toss("xRead() SQLITE_xxx error #"+rc,capi.sqlite3_wasm_rc_str(rc)); - } - blobList.push(heap.slice(iobuf, iobuf+nIobuf)); - } - return new Blob(blobList); - }catch(e){ - console.error('exportDbFileToBlob()',e); - stderr("exportDbFileToBlob():",e.message); - }finally{ - wasm.scopedAllocPop(scope); - } - }/*brokenExportDbFileToBlob()*/; - - const exportDbFileToBlob = function f(){ - if(!f._){ - f._ = sqlite3.capi.wasm.xWrap('fiddle_export_db', 'int', '*'); - } - const capi = sqlite3.capi; - const wasm = capi.wasm; - const blobList = []; - const heap = wasm.heap8u(); - const callback = wasm.installFunction('ipi', function(buf, n){ - blobList.push(heap.slice(buf, buf+n)); - return 0; - }); - try { - const rc = wasm.exports.fiddle_export_db( callback ); - if(rc) toss("DB export failed with code", capi.sqlite3_wasm_rc_str(rc)); - return new Blob(blobList); - }catch(e){ - console.error("exportDbFileToBlob():",e.message); - throw(e); - }finally{ - wasm.uninstallFunction(callback); - } - }/*exportDbFileToBlob()*/; self.onmessage = function f(ev){ ev = ev.data; @@ -344,11 +256,13 @@ stdout("Exporting",fn+"."); const fn2 = fn ? fn.split(/[/\\]/).pop() : null; try{ - if(!fn2) throw new Error("DB appears to be closed."); - exportDbFileToBlob().arrayBuffer().then((buffer)=>{ - wMsg('db-export',{filename: fn2, buffer}, [buffer]); - }); + if(!fn2) toss("DB appears to be closed."); + const buffer = sqlite3.capi.sqlite3_web_db_export( + Sqlite3Shell.dbHandle() + ); + wMsg('db-export',{filename: fn2, buffer: buffer.buffer}, [buffer.buffer]); }catch(e){ + console.error("Export failed:",e); /* Post a failure message so that UI elements disabled during the export can be re-enabled. */ wMsg('db-export',{ @@ -1,5 +1,5 @@ -C Merge\strunk\sinto\sfiddle-opfs\sbranch. -D 2022-10-01T13:45:14.035 +C Fiddle:\sfix\smakefile\sdependency\sissue\sand\sduplicate\sinclusion\sof\spost-js.js.\sReimplement\sdb\sexport\susing\ssqlite3_serialize(). +D 2022-10-01T16:01:41.242 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -485,13 +485,13 @@ F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba814 F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1ecc5ba9e64ef90a8b F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8de2d3f9b0b F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99 -F ext/wasm/api/sqlite3-api-glue.js ead29e6008ba148e7c67ad2bd928819dc72313ad2dcd34510cc61e089ee01c4e +F ext/wasm/api/sqlite3-api-glue.js b15a51b88aaa472d36bf82d5123dbfdafe8ddf6ca75fba934510e4a20bbe4adb F ext/wasm/api/sqlite3-api-oo1.js 9caed0757a5e039ed92467e827fd3ca347fa08f19fe086fcbdd14a4ebe9c2f01 F ext/wasm/api/sqlite3-api-opfs.js 1b097808b7b081b0f0700cf97d49ef19760e401706168edff9cd45cf9169f541 -F ext/wasm/api/sqlite3-api-prologue.js 04e0c929deeb28c9a2509d8004dfe3214992b988d277c4f50afa84953689c23c +F ext/wasm/api/sqlite3-api-prologue.js d7904da82691f68b9ffb081c072cf4725716154284a63447f2d1dce40ab4b85f F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 336389b23c9b83763177499e49a0967949c392b2f7d84fbbb52ad6678e159f18 +F ext/wasm/api/sqlite3-wasm.c 61c6bf8404a07f3d5f2861fbc1d3596474cecfc38801fb3a38c560fe847f79a5 F ext/wasm/batch-runner.html c363032aba7a525920f61f8be112a29459f73f07e46f0ba3b7730081a617826e F ext/wasm/batch-runner.js ce92650a6681586c89bef26ceae96674a55ca5a9727815202ca62e1a00ff5015 F ext/wasm/common/SqliteTestUtil.js 647bf014bd30bdd870a7e9001e251d12fc1c9ec9ce176a1004b838a4b33c5c05 @@ -503,9 +503,9 @@ F ext/wasm/demo-123.html aa281d33b7eefa755f3122b7b5a18f39a42dc5fb69c8879171bf14b F ext/wasm/demo-123.js 536579fd587974c2511c5bf82034b253d4fdeceabb726927ad7599ef6b7578e8 F ext/wasm/demo-kvvfs1.html 7d4f28873de67f51ac18c584b7d920825139866a96049a49c424d6f5a0ea5e7f F ext/wasm/demo-kvvfs1.js 105596bd2ccd0b1deb5fde8e99b536e8242d4bb5932fac0c8403ff3a6bc547e8 -F ext/wasm/fiddle.make d343d44c58bca06ac0ec0296207f6441560bff89f1e587bbaf843b73c2ca5d76 +F ext/wasm/fiddle.make 3f4efd62bc2a9c883bfcea52ae2755114a62d444d6d042df287f4aef301d6c6c F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 2a7107b06e5be3b9b063c340ec952f687e37ba6e0aa736b58c280dfb5e16625a +F ext/wasm/fiddle/fiddle-worker.js ebf6e95fe031738cd9f79d1ccdf6d80640a2884a43f7006a5e3459995f93069b F ext/wasm/fiddle/fiddle.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/index.html 63b370619e4f849ac76f1baed435c05edc29dbb6795bc7c1c935561ff667dd27 @@ -599,7 +599,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_kv.c cf02a39ab3271d237890c7de03a49e5ecc19c930b8f4ad6afd43651dc05c9400 +F src/os_kv.c 88872a68ba37e4ed4306eb6fa78d559dcc675761ce1f6458ba710e1c10a2d667 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902 @@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 000ef7059bfb54dc4f829b81a8d8c927c0382980218d8a3d60cd2c4d89151c90 b7da0bcdf70e53ab1ec00a0694e17c7429e23bc6eb3f39b622d06a930aa2f6a3 -R 5df8cd3132a0539ad9a559117068898c +P 64ebcbe41615a6d7776597564105ea7638e4a9095a764ea558c2620640429cf8 +R 624d71ac921985ea3965d7f2eda6af5a U stephan -Z 7ab2cbdb07e5b8371b60f5611bb9584c +Z 9261b853936f36a4eb842018d682a85c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cee93bde1..be02a9242 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64ebcbe41615a6d7776597564105ea7638e4a9095a764ea558c2620640429cf8
\ No newline at end of file +29db7de79232c21d19b91bb0fc253114e02e21dd9bf90619453dbe72a4c8bf7f
\ No newline at end of file diff --git a/src/os_kv.c b/src/os_kv.c index fb8eae627..8de687ff1 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -195,12 +195,12 @@ static void kvstorageMakeKey( ** be exported from the wasm file (but may still be used internally ** within the wasm file). ** -** The functions in this file (sqlite3-wasm.c) which require exporting -** are marked with this flag. They may also be added to any explicit -** build-time export list but need not be. All of these APIs are -** intended for use only within the project's own JS/WASM code, and -** not by client code, so an argument can be made for reducing their -** visibility by not including them in any build-time export lists. +** The functions in this filewhich require exporting are marked with +** this flag. They may also be added to any explicit build-time export +** list but need not be. All of these APIs are intended for use only +** within the project's own JS/WASM code, and not by client code, so +** an argument can be made for reducing their visibility by not +** including them in any build-time export lists. ** ** 2022-09-11: it's not yet _proven_ that this approach works in ** non-Emscripten builds. If not, such builds will need to export |