diff options
51 files changed, 4573 insertions, 1262 deletions
diff --git a/Makefile.in b/Makefile.in index d4d42c6b1..bb22039e8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1542,30 +1542,53 @@ wasm_dir_abs = $(TOP)/ext/wasm fiddle_dir = $(wasm_dir)/fiddle fiddle_dir_abs = $(TOP)/$(fiddle_dir) fiddle_module_js = $(fiddle_dir)/fiddle-module.js +sqlite3_wasm_c = $(wasm_dir)/api/sqlite3-wasm.c +$(sqlite3_wasm_c): sqlite3.c #emcc_opt = -O0 #emcc_opt = -O1 #emcc_opt = -O2 #emcc_opt = -O3 emcc_opt = -Oz +emcc_environment = web +# WASMFS/OPFS currently (2022-08-23) does not work with fiddle +# because (A) fiddle is primarily implemented as a Worker and (B) the +# Emscripten-based Worker loading process does not properly handle the +# case of nested Workers (necessary for it to load the WASMFS-specific +# Worker thread). +emcc_flags_wasmfs = +# To enable WASMFS/OPFS, uncomment these options: +#emcc_flags_wasmfs += -sWASMFS -pthread +#emcc_environment = web,worker +#emcc_flags_wasmfs += -DSQLITE_WASM_OPFS +#emcc_flags_wasmfs += -sPTHREAD_POOL_SIZE=2 +#emcc_flags_wasmfs += -sPTHREAD_POOL_SIZE_STRICT=2 +# (Thread pool settings may require tweaking.) +#/end of WASMFS/OPFS options. emcc_flags = $(emcc_opt) \ -sALLOW_TABLE_GROWTH \ -sABORTING_MALLOC \ -sSTRICT_JS \ - -sENVIRONMENT=web \ + -sENVIRONMENT=$(emcc_environment) \ -sMODULARIZE \ -sEXPORTED_RUNTIME_METHODS=@$(wasm_dir_abs)/EXPORTED_RUNTIME_METHODS.fiddle \ -sDYNAMIC_EXECUTION=0 \ --minify 0 \ -I. $(SHELL_OPT) \ - -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_UTF16 -DSQLITE_OMIT_DEPRECATED -$(fiddle_module_js): Makefile sqlite3.c shell.c \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + $(emcc_flags_wasmfs) +$(fiddle_module_js): Makefile $(sqlite3_wasm_c) shell.c \ $(wasm_dir)/EXPORTED_RUNTIME_METHODS.fiddle \ $(wasm_dir)/EXPORTED_FUNCTIONS.fiddle emcc -o $@ $(emcc_flags) \ -sEXPORT_NAME=initFiddleModule \ -sEXPORTED_FUNCTIONS=@$(wasm_dir_abs)/EXPORTED_FUNCTIONS.fiddle \ -DSQLITE_SHELL_FIDDLE \ - sqlite3.c shell.c + $(sqlite3_wasm_c) shell.c gzip < $@ > $@.gz gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz $(fiddle_dir)/fiddle.js.gz: $(fiddle_dir)/fiddle.js diff --git a/ext/wasm/EXPORTED_FUNCTIONS.fiddle b/ext/wasm/EXPORTED_FUNCTIONS.fiddle index b96ce4e67..602d61254 100644 --- a/ext/wasm/EXPORTED_FUNCTIONS.fiddle +++ b/ext/wasm/EXPORTED_FUNCTIONS.fiddle @@ -5,3 +5,5 @@ _fiddle_the_db _fiddle_db_arg _fiddle_db_filename _fiddle_reset_db +_sqlite3_wasm_init_opfs +_sqlite3_wasm_vfs_unlink diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 3133d1230..fd95b013c 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -18,6 +18,7 @@ else endif fiddle: $(MAKE) -C ../.. fiddle -e emcc_opt=$(fiddle_opt) +all: fiddle clean: $(MAKE) -C ../../ clean-fiddle @@ -32,6 +33,10 @@ dir.jacc := jaccwabyt dir.common := common CLEAN_FILES := *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ +sqlite3.c := $(dir.top)/sqlite3.c +$(sqlite3.c): + $(MAKE) -C $(dir.top) sqlite3.c + SQLITE_OPT = \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ @@ -45,10 +50,13 @@ SQLITE_OPT = \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED \ -DSQLITE_OMIT_UTF16 \ - -DSQLITE_THREADSAFE=0 + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 #SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5 -$(dir.top)/sqlite3.c: - $(MAKE) -C $(dir.top) sqlite3.c +# ^^^ MEMSYS5 is hypothetically useful for non-Emscripten builds but +# requires adding more infrastructure and fixing one spot in the +# sqlite3 internals which calls malloc() early on. # SQLITE_OMIT_LOAD_EXTENSION: if this is true, sqlite3_vfs::xDlOpen # and friends may be NULL. @@ -56,11 +64,14 @@ $(dir.top)/sqlite3.c: emcc_opt ?= -O0 .PHONY: release release: - $(MAKE) 'emcc_opt=-Os -g3' + $(MAKE) 'emcc_opt=-Os -g3 -flto' # ^^^^^ target-specific vars, e.g.: # release: emcc_opt=... # apparently only work for file targets, not PHONY targets? # +# ^^^ -flto improves runtime speed at -O0 considerably but doubles +# build time. +# # ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no # way around that except to use -g3, but -g3 causes the binary file # size to absolutely explode (approx. 5x larger). This minification @@ -127,11 +138,11 @@ sqlite3-api.jses := \ $(dir.jacc)/jaccwabyt.js \ $(dir.api)/sqlite3-api-glue.js \ $(dir.api)/sqlite3-api-oo1.js \ - $(dir.api)/sqlite3-api-worker.js \ - $(dir.api)/sqlite3-api-opfs.js \ - $(dir.api)/sqlite3-api-cleanup.js + $(dir.api)/sqlite3-api-worker1.js +#sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js -sqlite3-api.js := $(dir.api)/sqlite3-api.js +sqlite3-api.js := sqlite3-api.js CLEAN_FILES += $(sqlite3-api.js) $(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) @echo "Making $@..." @@ -141,7 +152,7 @@ $(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) echo "/* END FILE: $$i */"; \ done > $@ -post-js.js := $(dir.api)/post-js.js +post-js.js := post-js.js CLEAN_FILES += $(post-js.js) post-jses := \ $(dir.api)/post-js-header.js \ @@ -158,9 +169,12 @@ $(post-js.js): $(post-jses) $(MAKEFILE) ######################################################################## -# emcc flags for .c/.o/.wasm. +# emcc flags for .c/.o/.wasm/.js. emcc.flags = #emcc.flags += -v # _very_ loud but also informative about what it's doing +# -g is needed to keep -O2 and higher from creating broken JS via +# minification. +emcc.flags += -g ######################################################################## # emcc flags for .c/.o. @@ -172,41 +186,55 @@ emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT) ######################################################################## # emcc flags specific to building the final .js/.wasm file... emcc.jsflags := -fPIC +emcc.jsflags := --minify 0 emcc.jsflags += --no-entry -emcc.jsflags += -sENVIRONMENT=web emcc.jsflags += -sMODULARIZE emcc.jsflags += -sSTRICT_JS emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api -emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY +emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack + # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY + # allocateUTF8OnStack => for kvvp internals emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY -#emcc.jsflags += -sINITIAL_MEMORY=13107200 +emcc.environment := -sENVIRONMENT=web +ENABLE_WASMFS ?= 0 +ifneq (0,$(ENABLE_WASMFS)) + emcc.cflags += -pthread + emcc.jsflags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2 + emcc.cflags += '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' + emcc.environment := $(emcc.environment),worker + emcc.jsflags += -sINITIAL_MEMORY=128450560 +else + emcc.jsflags += -sALLOW_MEMORY_GROWTH + # emcc: warning: USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code + # slowly, see https://github.com/WebAssembly/design/issues/1271 + # [-Wpthreads-mem-growth] + emcc.jsflags += -sINITIAL_MEMORY=13107200 + #emcc.jsflags += -sINITIAL_MEMORY=64225280 + # ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js +endif +emcc.jsflags += $(emcc.environment) #emcc.jsflags += -sTOTAL_STACK=4194304 emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. -emcc.jsflags +=--post-js=$(post-js.js) +emcc.jsflags += --post-js=$(post-js.js) #emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() #emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS #emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API #emcc.jsflags += -sABORTING_MALLOC -emcc.jsflags += -sALLOW_MEMORY_GROWTH emcc.jsflags += -sALLOW_TABLE_GROWTH emcc.jsflags += -Wno-limited-postlink-optimizations # ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -#emcc.jsflags += -sMALLOC=emmalloc -#emcc.jsflags += -sMALLOC=dlmalloc # a good 8k larger than emmalloc #emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why -#emcc.jsflags += --import=foo_bar -#emcc.jsflags += --no-gc-sections # https://lld.llvm.org/WebAssembly.html emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --allow-undefined emcc.jsflags += --import-undefined #emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic -#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined +#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all enable_bigint ?= 1 ifneq (0,$(enable_bigint)) @@ -222,22 +250,41 @@ emcc.jsflags += -sMEMORY64=0 # new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) # # because ptr is now a BigInt, so is invalid for passing to arguments -# which have strict must-be-a-number requirements. +# which have strict must-be-a-Number requirements. ######################################################################## -sqlite3.js := $(dir.api)/sqlite3.js -sqlite3.wasm := $(dir.api)/sqlite3.wasm -$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT) -$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c +######################################################################## +# -sSINGLE_FILE: +# https://github.com/emscripten-core/emscripten/blob/main/src/settings.js#L1704 +# -sSINGLE_FILE=1 would be really nice but we have to build with -g +# for -O2 and higher to work (else minification breaks the code) and +# cannot wasm-strip the binary before it gets encoded into the JS +# file. The result is that the generated JS file is, because of the -g +# debugging info, _huge_. +######################################################################## + +######################################################################## +# Maintenance reminder: the output .js and .wasm files of emcc must be +# in _this_ dir, rather than a subdir, or else parts of the generated +# code get confused and cannot load property (namely, the +# sqlite3.worker.js generated in conjunction with -sWASMFS). +sqlite3.js := sqlite3.js +sqlite3.wasm := sqlite3.wasm +sqlite3-wasm.o := $(dir.api)/sqlite3-wasm.o +$(sqlite3-wasm.o): emcc.cflags += $(SQLITE_OPT) +$(sqlite3-wasm.o): $(dir.top)/sqlite3.c $(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT) -sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \ - $(dir.jacc)/jaccwabyt_test.c -# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test -# apps. However, we want to test the release builds with those apps, -# so we cannot simply elide that file in release builds. That -# component is critical to the VFS bindings so needs to be tested -# along with the core APIs. +sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c +jaccwabyt_test.c := $(dir.jacc)/jaccwabyt_test.c +# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test apps, +# so we don't really want to include it in release builds. However, we +# want to test the release builds with those apps, so we cannot simply +# elide that file in release builds. That component is critical to the +# VFS bindings so needs to be tested along with the core APIs. +ifneq (,$(filter -sWASMFS,$(emcc.jsflags))) + $(sqlite3-wasm.o): emcc.cflags+=-DSQLITE_WASM_OPFS +endif define WASM_C_COMPILE $(1).o := $$(subst .c,.o,$(1)) sqlite3.wasm.obj += $$($(1).o) @@ -245,12 +292,12 @@ $$($(1).o): $$(MAKEFILE) $(1) $$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@ CLEAN_FILES += $$($(1).o) endef -$(foreach c,$(sqlite3.wasm.c),$(eval $(call WASM_C_COMPILE,$(c)))) +$(foreach c,$(sqlite3-wasm.c) $(jaccwabyt_test.c),$(eval $(call WASM_C_COMPILE,$(c)))) $(sqlite3.js): $(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ EXPORTED_FUNCTIONS.api \ $(post-js.js) - $(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) + $(emcc.bin) -o $(sqlite3.js) $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) chmod -x $(sqlite3.wasm) ifneq (,$(wasm-strip)) $(wasm-strip) $(sqlite3.wasm) @@ -259,10 +306,81 @@ endif CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) all: $(sqlite3.js) +wasm: $(sqlite3.js) # End main Emscripten-based module build ######################################################################## -include kvvfs.make +######################################################################## +# batch-runner.js... +dir.sql := sql +speedtest1 := ../../speedtest1 +speedtest1.c := ../../test/speedtest1.c +speedtest1.sql := $(dir.sql)/speedtest1.sql +$(speedtest1): + $(MAKE) -C ../.. speedtest1 +$(speedtest1.sql): $(speedtest1) + $(speedtest1) --script $@ +batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql + bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql + ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@ +clean-batch: + rm -f batch-runner.list $(dir.sql)/speedtest1*.sql +# ^^^ we don't do this along with 'clean' because we clean/rebuild on +# a regular basis with different -Ox flags and rebuilding the batch +# pieces each time is an unnecessary time sink. +batch: batch-runner.list +all: batch +# end batch-runner.js +######################################################################## +# speedtest1.js... +emcc.speedtest1-flags := -g $(emcc_opt) +ifneq (0,$(ENABLE_WASMFS)) + emcc.speedtest1-flags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2 + emcc.speedtest1-flags += -DSQLITE_WASM_OPFS +endif +emcc.speedtest1-flags += -sINVOKE_RUN=0 +#emcc.speedtest1-flags += --no-entry +emcc.speedtest1-flags += -flto +emcc.speedtest1-flags += -sABORTING_MALLOC +emcc.speedtest1-flags += -sINITIAL_MEMORY=128450560 +emcc.speedtest1-flags += -sSTRICT_JS +emcc.speedtest1-flags += $(emcc.environment) +emcc.speedtest1-flags += -sMODULARIZE +emcc.speedtest1-flags += -sEXPORT_NAME=sqlite3Speedtest1InitModule +emcc.speedtest1-flags += -Wno-limited-postlink-optimizations +emcc.speedtest1-flags += -sEXPORTED_FUNCTIONS=_main,_malloc,_free,_sqlite3_wasm_vfs_unlink,_sqlite3_wasm_init_opfs +emcc.speedtest1-flags += -sDYNAMIC_EXECUTION=0 +emcc.speedtest1-flags += --minify 0 + +speedtest1.js := speedtest1.js +speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) +$(speedtest1.js): emcc.cflags+= +# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against +# the latter (predictably) results in a slightly faster binary, but we're +# close enough to the target speed requirements that the 500ms makes a +# difference. +$(speedtest1.js): $(speedtest1.c) $(sqlite3-wasm.c) $(MAKEFILE) + $(emcc.bin) \ + $(emcc.speedtest1-flags) \ + -I. -I$(dir.top) \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + -DSQLITE_SPEEDTEST1_WASM \ + -o $@ $(speedtest1.c) $(sqlite3-wasm.c) -lm +ifneq (,$(wasm-strip)) + $(wasm-strip) $(speedtest1.wasm) +endif + ls -la $@ $(speedtest1.wasm) + +speedtest1: $(speedtest1.js) +all: $(speedtest1.js) +CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) +# end speedtest1.js +######################################################################## ######################################################################## # fiddle_remote is the remote destination for the fiddle app. It @@ -286,3 +404,5 @@ push-fiddle: $(fiddle_files) rsync -va fiddle/ $(fiddle_remote) # end fiddle remote push ######################################################################## + +include kvvfs.make diff --git a/ext/wasm/README.md b/ext/wasm/README.md index 1702e1f42..f95001359 100644 --- a/ext/wasm/README.md +++ b/ext/wasm/README.md @@ -10,6 +10,7 @@ below for Linux environments: ``` # Clone the emscripten repository: +$ sudo apt install git $ git clone https://github.com/emscripten-core/emsdk.git $ cd emsdk @@ -74,6 +75,32 @@ from 2022-05-17 or newer so that it recognizes the `.wasm` file extension and responds with the mimetype `application/wasm`, as the WASM loader is pedantic about that detail. +# Testing on a remote machine that is accessed via SSH + +*NB: The following are developer notes, last validated on 2022-08-18* + + * Remote: Install git, emsdk, and althttpd + * Use a [version of althttpd](https://sqlite.org/althttpd/timeline?r=enable-atomics) + that adds HTTP reply header lines to enable SharedArrayBuffers. These header + lines are required: +``` + Cross-Origin-Opener-Policy: same-origin + Cross-Origin-Embedder-Policy: require-corp +``` + * Remote: Install the SQLite source tree. CD to ext/wasm + * Remote: "`make`" to build WASM + * Remote: althttpd --port 8080 --popup + * Local: ssh -L 8180:localhost:8080 remote + * Local: Point your web-browser at http://localhost:8180/testing1.html + +In order to enable [SharedArrayBuffers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer), +the web-browser requires that the two extra Cross-Origin lines be present +in HTTP reply headers and that the request must come from "localhost". +Since the web-server is on a different machine from +the web-broser, the localhost requirement means that the connection must be tunneled +using SSH. + + # Known Quirks and Limitations diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 8f103c7c0..aead79e50 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -66,7 +66,5 @@ _sqlite3_value_text _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register -_sqlite3_wasm_db_error -_sqlite3_wasm_enum_json _malloc _free diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 43d2b0dd5..9000697b2 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -60,17 +60,17 @@ browser client: high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. That said, it is not a "required component" and can be elided from builds which do not want it. -- `sqlite3-api-worker.js`\ +- `sqlite3-api-worker1.js`\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - `sqlite3-worker.js`\ + - `sqlite3-worker1.js`\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module - and runs the Worker API which is implemented in - `sqlite3-api-worker.js`. + and runs the Worker #1 API which is implemented in + `sqlite3-api-worker1.js`. - `sqlite3-api-opfs.js`\ is an in-development/experimental sqlite3 VFS wrapper, the goal of which being to use Google Chrome's Origin-Private FileSystem (OPFS) diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index a2f921a5d..1b57cdc5d 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -11,34 +11,45 @@ *********************************************************************** This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other files so that it can clean - up any global systems temporarily used for setting up the API's - various subsystems. + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. */ 'use strict'; -self.sqlite3.postInit.forEach( - self.importScripts/*global is a Worker*/ - ? function(f){ - /** We try/catch/report for the sake of failures which happen in - a Worker, as those exceptions can otherwise get completely - swallowed, leading to confusing downstream errors which have - nothing to do with this failure. */ - try{ f(self, self.sqlite3) } - catch(e){ - console.error("Error in postInit() function:",e); - throw e; - } - } - : (f)=>f(self, self.sqlite3) -); -delete self.sqlite3.postInit; -if(self.location && +self.location.port > 1024){ - console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); - self.S = self.sqlite3; +if('undefined' !== typeof Module){ // presumably an Emscripten build + /** + Install a suitable default configuration for sqlite3ApiBootstrap(). + */ + const SABC = self.sqlite3ApiBootstrap.defaultConfig; + SABC.Module = Module /* ==> Currently needs to be exposed here for test code. NOT part + of the public API. */; + SABC.exports = Module['asm']; + SABC.memory = Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */; + + /** + For current (2022-08-22) purposes, automatically call + sqlite3ApiBootstrap(). That decision will be revisited at some + point, as we really want client code to be able to call this to + configure certain parts. Clients may modify + self.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(). + */ + //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); + const sqlite3 = self.sqlite3ApiBootstrap(); + delete self.sqlite3ApiBootstrap; + + if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); + self.S = sqlite3; + } + + /* Clean up temporary references to our APIs... */ + delete sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; + //console.warn("Module.sqlite3 =",Module.sqlite3); + Module.sqlite3 = sqlite3 /* Currently needed by test code and sqlite3-worker1.js */; +}else{ + console.warn("This is not running in an Emscripten module context, so", + "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "of config info for the WASM environment.", + "It must be called manually."); } -/* Clean up temporary global-scope references to our APIs... */ -self.sqlite3.config.Module.sqlite3 = self.sqlite3 -/* ^^^^ Currently needed by test code and Worker API setup */; -delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; -delete self.sqlite3 /* clean up our global-scope reference */; -//console.warn("Module.sqlite3 =",Module.sqlite3); diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e962c93b6..3a9e8803c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -16,23 +16,9 @@ initializes the main API pieces so that the downstream components (e.g. sqlite3-api-oo1.js) have all that they need. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - - self.sqlite3 = self.sqlite3ApiBootstrap({ - Module: Module /* ==> Emscripten-style Module object. Currently - needs to be exposed here for test code. NOT part - of the public API. */, - exports: Module['asm'], - memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */, - bigIntEnabled: !!self.BigInt64Array, - allocExportName: 'malloc', - deallocExportName: 'free' - }); - delete self.sqlite3ApiBootstrap; - - const sqlite3 = self.sqlite3; const capi = sqlite3.capi, wasm = capi.wasm, util = capi.util; self.WhWasmUtilInstaller(capi.wasm); delete self.WhWasmUtilInstaller; @@ -57,7 +43,7 @@ return oldP(v); }; wasm.xWrap.argAdapter('.pointer', adapter); - } + } /* ".pointer" xWrap() argument adapter */ // WhWasmUtil.xWrap() bindings... { @@ -77,8 +63,11 @@ for(const e of wasm.bindingSignatures){ capi[e[0]] = wasm.xWrap.apply(null, e); } + for(const e of wasm.bindingSignatures.wasm){ + capi.wasm[e[0]] = wasm.xWrap.apply(null, e); + } - /* For functions which cannot work properly unless + /* For C API functions which cannot work properly unless wasm.bigIntEnabled is true, install a bogus impl which throws if called when bigIntEnabled is false. */ const fI64Disabled = function(fname){ @@ -128,7 +117,7 @@ */ __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', "int", ["sqlite3*", "string", - "int"/*MUST always be negative*/, + "int"/*ignored for this impl!*/, "int", "**", "**"/*MUST be 0 or null or undefined!*/]); /** @@ -148,19 +137,10 @@ /* Documented in the api object's initializer. */ capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ - /* 2022-07-08: xWrap() 'string' arg handling may be able do this - special-case handling for us. It needs to be tested. Or maybe - not: we always want to treat pzTail as null when passed a - non-pointer SQL string and the argument adapters don't have - enough state to know that. Maybe they could/should, by passing - the currently-collected args as an array as the 2nd arg to the - argument adapters? Or maybe we collect all args in an array, - pass that to an optional post-args-collected callback, and give - it a chance to manipulate the args before we pass them on? */ if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); switch(typeof sql){ case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); - case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); + case 'number': return __prepare.full(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail); default: return util.sqlite3_wasm_db_error( pDb, capi.SQLITE_MISUSE, @@ -207,5 +187,4 @@ capi[s.name] = sqlite3.StructBinder(s); } } - -})(self); +}); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 9e5473396..af179d1fe 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -14,10 +14,9 @@ WASM build. It requires that sqlite3-api-glue.js has already run and it installs its deliverable as self.sqlite3.oo1. */ -(function(self){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = (...args)=>{throw new Error(args.join(' '))}; - const sqlite3 = self.sqlite3 || toss("Missing main sqlite3 object."); const capi = sqlite3.capi, util = capi.util; /* What follows is colloquially known as "OO API #1". It is a binding of the sqlite3 API which is designed to be run within @@ -59,14 +58,86 @@ enabling clients to unambiguously identify such exceptions. */ class SQLite3Error extends Error { + /** + Constructs this object with a message equal to all arguments + concatenated with a space between each one. + */ constructor(...args){ - super(...args); + super(args.join(' ')); this.name = 'SQLite3Error'; } }; - const toss3 = (...args)=>{throw new SQLite3Error(args)}; + const toss3 = (...args)=>{throw new SQLite3Error(...args)}; sqlite3.SQLite3Error = SQLite3Error; + // Documented in DB.checkRc() + const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + throw new SQLite3Error( + "sqlite result code",sqliteResultCode+":", + (dbPtr + ? capi.sqlite3_errmsg(dbPtr) + : capi.sqlite3_errstr(sqliteResultCode)) + ); + } + }; + + /** + A proxy for DB class constructors. It must be called with the + being-construct DB object as its "this". + */ + const dbCtorHelper = function ctor(fn=':memory:', flags='c', vfsName){ + if(!ctor._name2vfs){ + // Map special filenames which we handle here (instead of in C) + // to some helpful metadata... + ctor._name2vfs = Object.create(null); + const isWorkerThread = (self.window===self /*===running in main window*/) + ? false + : (n)=>toss3("The VFS for",n,"is only available in the main window thread.") + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', + filename: isWorkerThread || (()=>'session') + }; + } + if('string'!==typeof fn){ + toss3("Invalid filename for DB constructor."); + } + const vfsCheck = ctor._name2vfs[fn]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = vfsCheck.filename(fn); + } + let ptr, oflags = 0; + if( flags.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flags.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = capi.wasm.scopedAllocPush(); + try { + const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; + const pVfsName = vfsName ? capi.wasm.scopedAllocCString(vfsName) : 0; + const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName); + ptr = capi.wasm.getPtrValue(ppDb); + checkSqlite3Rc(ptr, rc); + }catch( e ){ + if( ptr ) capi.sqlite3_close_v2(ptr); + throw e; + }finally{ + capi.wasm.scopedAllocPop(stack); + } + this.filename = fn; + __ptrMap.set(this, ptr); + __stmtMap.set(this, Object.create(null)); + __udfMap.set(this, Object.create(null)); + }; + /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. @@ -80,39 +151,48 @@ not resolve to real filenames, but "" uses an on-storage temporary database and requires that the VFS support that. - The db is currently opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API will change in the future - permit the caller to provide those flags via an additional - argument. + The second argument specifies the open/create mode for the + database. It must be string containing a sequence of letters (in + any order, but case sensitive) specifying the mode: + + - "c" => create if it does not exist, else fail if it does not + exist. Implies the "w" flag. + + - "w" => write. Implies "r": a db cannot be write-only. + + - "r" => read-only if neither "w" nor "c" are provided, else it + is ignored. + + If "w" is not provided, the db is implicitly read-only, noting that + "rc" is meaningless + + Any other letters are currently ignored. The default is + "c". These modes are ignored for the special ":memory:" and "" + names. + + The final argument is analogous to the final argument of + sqlite3_open_v2(): the name of an sqlite3 VFS. Pass a falsy value, + or not at all, to use the default. If passed a value, it must + be the string name of a VFS For purposes of passing a DB instance to C-style sqlite3 - functions, its read-only `pointer` property holds its `sqlite3*` - pointer value. That property can also be used to check whether - this DB instance is still open. + functions, the DB object's read-only `pointer` property holds its + `sqlite3*` pointer value. That property can also be used to check + whether this DB instance is still open. + + + EXPERIMENTAL: in the main window thread, the filenames + ":localStorage:" and ":sessionStorage:" are special: they cause + the db to use either localStorage or sessionStorage for storing + the database. In this mode, only a single database is permitted + in each storage object. This feature is experimental and subject + to any number of changes (including outright removal). This + support requires a specific build of sqlite3, the existence of + which can be determined at runtime by checking for a non-0 return + value from sqlite3.capi.sqlite3_vfs_find("kvvfs"). */ - const DB = function ctor(fn=':memory:'){ - if('string'!==typeof fn){ - toss3("Invalid filename for DB constructor."); - } - const stack = capi.wasm.scopedAllocPush(); - let ptr; - try { - const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; - const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE - | capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_EXRESCODE, null); - ptr = capi.wasm.getMemValue(ppDb, '*'); - ctor.checkRc(ptr, rc); - }catch(e){ - if(ptr) capi.sqlite3_close_v2(ptr); - throw e; - } - finally{capi.wasm.scopedAllocPop(stack);} - this.filename = fn; - __ptrMap.set(this, ptr); - __stmtMap.set(this, Object.create(null)); - __udfMap.set(this, Object.create(null)); + const DB = function ctor(fn=':memory:', flags='c', vfsName){ + dbCtorHelper.apply(this, Array.prototype.slice.call(arguments)); }; /** @@ -141,6 +221,15 @@ For purposes of passing a Stmt instance to C-style sqlite3 functions, its read-only `pointer` property holds its `sqlite3_stmt*` pointer value. + + Other non-function properties include: + + - `db`: the DB object which created the statement. + + - `columnCount`: the number of result columns in the query, or 0 for + queries which cannot return results. + + - `parameterCount`: the number of bindable paramters in the query. */ const Stmt = function(){ if(BindTypes!==arguments[2]){ @@ -163,7 +252,7 @@ Reminder: this will also fail after the statement is finalized but the resulting error will be about an out-of-bounds column - index. + index rather than a statement-is-finalized error. */ const affirmColIndex = function(stmt,ndx){ if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ @@ -173,16 +262,20 @@ }; /** - Expects to be passed (arguments) from DB.exec() and - DB.execMulti(). Does the argument processing/validation, throws - on error, and returns a new object on success: + Expects to be passed the `arguments` object from DB.exec(). Does + the argument processing/validation, throws on error, and returns + a new object on success: { sql: the SQL, opt: optionsObj, cbArg: function} - cbArg is only set if the opt.callback is set, in which case - it's a function which expects to be passed the current Stmt - and returns the callback argument of the type indicated by - the input arguments. + The opt object is a normalized copy of any passed to this + function. The sql will be converted to a string if it is provided + in one of the supported non-string formats. + + cbArg is only set if the opt.callback or opt.resultRows are set, + in which case it's a function which expects to be passed the + current Stmt and returns the callback argument of the type + indicated by the input arguments. */ const parseExecArgs = function(args){ const out = Object.create(null); @@ -194,6 +287,8 @@ }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; out.sql = out.opt.sql; + }else if(Array.isArray(args[0])){ + out.sql = args[0]; } break; case 2: @@ -211,14 +306,14 @@ } if(out.opt.callback || out.opt.resultRows){ switch((undefined===out.opt.rowMode) - ? 'stmt' : out.opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get({}); break; + ? 'array' : out.opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; case 'array': out.cbArg = (stmt)=>stmt.get([]); break; case 'stmt': if(Array.isArray(out.opt.resultRows)){ - toss3("Invalid rowMode for resultRows array: must", + toss3("exec(): invalid rowMode for a resultRows array: must", "be one of 'array', 'object',", - "or a result column number."); + "a result column number, or column name reference."); } out.cbArg = (stmt)=>stmt; break; @@ -226,6 +321,19 @@ if(util.isInt32(out.opt.rowMode)){ out.cbArg = (stmt)=>stmt.get(out.opt.rowMode); break; + }else if('string'===typeof out.opt.rowMode && out.opt.rowMode.length>1){ + /* "$X", ":X", and "@X" fetch column named "X" (case-sensitive!) */ + const prefix = out.opt.rowMode[0]; + if(':'===prefix || '@'===prefix || '$'===prefix){ + out.cbArg = function(stmt){ + const rc = stmt.get(this.obj)[this.colName]; + return (undefined===rc) ? toss3("exec(): unknown result column:",this.colName) : rc; + }.bind({ + obj:Object.create(null), + colName: out.opt.rowMode.substr(1) + }); + break; + } } toss3("Invalid rowMode:",out.opt.rowMode); } @@ -234,24 +342,17 @@ }; /** - Expects to be given a DB instance or an `sqlite3*` pointer, and an - sqlite3 API result code. If the result code is not falsy, this - function throws an SQLite3Error with an error message from - sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's - passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." The - various non-0 non-error codes need to be checked for in client - code where they are expected. + Expects to be given a DB instance or an `sqlite3*` pointer (may + be null) and an sqlite3 API result code. If the result code is + not falsy, this function throws an SQLite3Error with an error + message from sqlite3_errmsg(), using dbPtr as the db handle, or + sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a + non-error code like SQLITE_ROW or SQLITE_DONE, it will still + throw but the error string might be "Not an error." The various + non-0 non-error codes need to be checked for in + client code where they are expected. */ - DB.checkRc = function(dbPtr, sqliteResultCode){ - if(sqliteResultCode){ - if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; - throw new SQLite3Error([ - "sqlite result code",sqliteResultCode+":", - capi.sqlite3_errmsg(dbPtr) || "Unknown db error." - ].join(' ')); - } - }; + DB.checkRc = checkSqlite3Rc; DB.prototype = { /** @@ -300,26 +401,24 @@ } }, /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. + Similar to this.filename but will return NULL for special names + like ":memory:". Not of much use until we have filesystem + support. Throws if the DB has been closed. If passed an + argument it then it will return the filename of the ATTACHEd db + with that name, else it assumes a name of `main`. */ - fileName: function(dbName){ - return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); + fileName: function(dbName='main'){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); }, /** Returns true if this db instance has a name which resolves to a file. If the name is "" or ":memory:", it resolves to false. Note that it is not aware of the peculiarities of URI-style names and a URI-style name for a ":memory:" db will fool it. + Returns false if this db is closed. */ hasFilename: function(){ - const fn = this.filename; - if(!fn || ':memory'===fn) return false; - return true; + return this.filename && ':memory'!==this.filename; }, /** Returns the name of the given 0-based db number, as documented @@ -343,9 +442,8 @@ required to check `stmt.pointer` after calling `prepare()` in order to determine whether the Stmt instance is empty or not. Long-time practice (with other sqlite3 script bindings) - suggests that the empty-prepare case is sufficiently rare (and - useless) that supporting it here would simply hurt overall - usability. + suggests that the empty-prepare case is sufficiently rare that + supporting it here would simply hurt overall usability. */ prepare: function(sql){ affirmDbOpen(this); @@ -354,7 +452,7 @@ try{ ppStmt = capi.wasm.scopedAllocPtr()/* output (sqlite3_stmt**) arg */; DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); - pStmt = capi.wasm.getMemValue(ppStmt, '*'); + pStmt = capi.wasm.getPtrValue(ppStmt); } finally {capi.wasm.scopedAllocPop(stack)} if(!pStmt) toss3("Cannot prepare empty SQL."); @@ -363,70 +461,6 @@ return stmt; }, /** - This function works like execMulti(), and takes most of the - same arguments, but is more efficient (performs much less - work) when the input SQL is only a single statement. If - passed a multi-statement SQL, it only processes the first - one. - - This function supports the following additional options not - supported by execMulti(): - - - .multi: if true, this function acts as a proxy for - execMulti() and behaves identically to that function. - - - .columnNames: if this is an array and the query has - result columns, the array is passed to - Stmt.getColumnNames() to append the column names to it - (regardless of whether the query produces any result - rows). If the query has no result columns, this value is - unchanged. - - The following options to execMulti() are _not_ supported by - this method (they are simply ignored): - - - .saveSql - */ - exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ - affirmDbOpen(this); - const arg = parseExecArgs(arguments); - if(!arg.sql) return this; - else if(arg.opt.multi){ - return this.execMulti(arg, undefined, BindTypes); - } - const opt = arg.opt; - let stmt, rowTarget; - try { - if(Array.isArray(opt.resultRows)){ - rowTarget = opt.resultRows; - } - stmt = this.prepare(arg.sql); - if(stmt.columnCount && Array.isArray(opt.columnNames)){ - stmt.getColumnNames(opt.columnNames); - } - if(opt.bind) stmt.bind(opt.bind); - if(opt.callback || rowTarget){ - while(stmt.step()){ - const row = arg.cbArg(stmt); - if(rowTarget) rowTarget.push(row); - if(opt.callback){ - stmt._isLocked = true; - opt.callback(row, stmt); - stmt._isLocked = false; - } - } - }else{ - stmt.step(); - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - } - return this; - }/*exec()*/, - /** Executes one or more SQL statements in the form of a single string. Its arguments must be either (sql,optionsObject) or (optionsObject). In the latter case, optionsObject.sql @@ -440,92 +474,113 @@ The optional options object may contain any of the following properties: - - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string, Uint8Array, or an - array of strings (in which case they're concatenated - together as-is, with no separator between elements, - before evaluation). - - - .bind = a single value valid as an argument for - Stmt.bind(). This is ONLY applied to the FIRST non-empty - statement in the SQL which has any bindable - parameters. (Empty statements are skipped entirely.) - - - .callback = a function which gets called for each row of - the FIRST statement in the SQL which has result - _columns_, but only if that statement has any result - _rows_. The second argument passed to the callback is - always the current Stmt object (so that the caller may - collect column names, or similar). The first argument - passed to the callback defaults to the current Stmt - object but may be changed with ... - - - .rowMode = either a string describing what type of argument - should be passed as the first argument to the callback or an - integer representing a result column index. A `rowMode` of - 'object' causes the results of `stmt.get({})` to be passed to - the `callback` and/or appended to `resultRows`. A value of - 'array' causes the results of `stmt.get([])` to be passed to - passed on. A value of 'stmt' is equivalent to the default, - passing the current Stmt to the callback (noting that it's - always passed as the 2nd argument), but this mode will trigger - an exception if `resultRows` is an array. If `rowMode` is an - integer, only the single value from that result column will be - passed on. Any other value for the option triggers an - exception. + - `.sql` = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an array + of strings. In the latter case they're concatenated together + as-is, _with no separator_ between elements, before evaluation. + The array form is often simpler for long hand-written queries. + + - `.bind` = a single value valid as an argument for + Stmt.bind(). This is _only_ applied to the _first_ non-empty + statement in the SQL which has any bindable parameters. (Empty + statements are skipped entirely.) - - .resultRows: if this is an array, it functions similarly to - the `callback` option: each row of the result set (if any) of - the FIRST first statement which has result _columns_ is - appended to the array in the format specified for the `rowMode` - option, with the exception that the only legal values for - `rowMode` in this case are 'array' or 'object', neither of - which is the default. It is legal to use both `resultRows` and - `callback`, but `resultRows` is likely much simpler to use for - small data sets and can be used over a WebWorker-style message - interface. execMulti() throws if `resultRows` is set and - `rowMode` is 'stmt' (which is the default!). - - - saveSql = an optional array. If set, the SQL of each + - `.saveSql` = an optional array. If set, the SQL of each executed statement is appended to this array before the - statement is executed (but after it is prepared - we - don't have the string until after that). Empty SQL - statements are elided. - - See also the exec() method, which is a close cousin of this - one. - - ACHTUNG #1: The callback MUST NOT modify the Stmt - object. Calling any of the Stmt.get() variants, - Stmt.getColumnName(), or similar, is legal, but calling - step() or finalize() is not. Routines which are illegal - in this context will trigger an exception. - - ACHTUNG #2: The semantics of the `bind` and `callback` - options may well change or those options may be removed - altogether for this function (but retained for exec()). - Generally speaking, neither bind parameters nor a callback - are generically useful when executing multi-statement SQL. + statement is executed (but after it is prepared - we don't have + the string until after that). Empty SQL statements are elided. + + ================================================================== + The following options apply _only_ to the _first_ statement + which has a non-zero result column count, regardless of whether + the statement actually produces any result rows. + ================================================================== + + - `.callback` = a function which gets called for each row of + the result set, but only if that statement has any result + _rows_. The callback's "this" is the options object. The second + argument passed to the callback is always the current Stmt + object (so that the caller may collect column names, or + similar). The 2nd argument to the callback is always the Stmt + instance, as it's needed if the caller wants to fetch the + column names or some such (noting that they could also be + fetched via `this.columnNames`, if the client provides the + `columnNames` option). + + ACHTUNG: The callback MUST NOT modify the Stmt object. Calling + any of the Stmt.get() variants, Stmt.getColumnName(), or + similar, is legal, but calling step() or finalize() is + not. Routines which are illegal in this context will trigger an + exception. + + The first argument passed to the callback defaults to an array of + values from the current result row but may be changed with ... + + - `.rowMode` = specifies the type of he callback's first argument. + It may be any of... + + A) A string describing what type of argument should be passed + as the first argument to the callback: + + A.1) `'array'` (the default) causes the results of + `stmt.get([])` to be passed to passed on and/or appended to + `resultRows`. + + A.2) `'object'` causes the results of + `stmt.get(Object.create(null))` to be passed to the + `callback` and/or appended to `resultRows`. Achtung: an SQL + result may have multiple columns with identical names. In + that case, the right-most column will be the one set in this + object! + + A.3) `'stmt'` causes the current Stmt to be passed to the + callback, but this mode will trigger an exception if + `resultRows` is an array because appending the statement to + the array would be unhelpful. + + B) An integer, indicating a zero-based column in the result + row. Only that one single value will be passed on. + + C) A string with a minimum length of 2 and leading character of + ':', '$', or '@' will fetch the row as an object, extract that + one field, and pass that field's value to the callback. Note + that these keys are case-sensitive so must match the case used + in the SQL. e.g. `"select a A from t"` with a `rowMode` of '$A' + would work but '$a' would not. A reference to a column not in + the result set will trigger an exception on the first row (as + the check is not performed until rows are fetched). + + Any other `rowMode` value triggers an exception. + + - `.resultRows`: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any), + with the exception that the `rowMode` 'stmt' is not legal. It + is legal to use both `resultRows` and `callback`, but + `resultRows` is likely much simpler to use for small data sets + and can be used over a WebWorker-style message interface. + exec() throws if `resultRows` is set and `rowMode` is 'stmt'. + + - `.columnNames`: if this is an array, the column names of the + result set are stored in this array before the callback (if + any) is triggered (regardless of whether the query produces any + result rows). If no statement has result columns, this value is + unchanged. Achtung: an SQL result may have multiple columns + with identical names. */ - execMulti: function(/*(sql [,obj]) || (obj)*/){ + exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); const wasm = capi.wasm; - const arg = (BindTypes===arguments[2] - /* ^^^ Being passed on from exec() */ - ? arguments[0] : parseExecArgs(arguments)); - if(!arg.sql) return this; + const arg = parseExecArgs(arguments); + if(!arg.sql){ + return (''===arg.sql) ? this : toss3("exec() requires an SQL string."); + } const opt = arg.opt; const callback = opt.callback; - const resultRows = (Array.isArray(opt.resultRows) + let resultRows = (Array.isArray(opt.resultRows) ? opt.resultRows : undefined); - if(resultRows && 'stmt'===opt.rowMode){ - toss3("rowMode 'stmt' is not valid in combination", - "with a resultRows array."); - } - let rowMode = (((callback||resultRows) && (undefined!==opt.rowMode)) - ? opt.rowMode : undefined); let stmt; let bind = opt.bind; + let evalFirstResult = !!(arg.cbArg || opt.columnNames) /* true to evaluate the first result-returning query */; const stack = wasm.scopedAllocPush(); try{ const isTA = util.isSQLableTypedArray(arg.sql) @@ -544,21 +599,21 @@ if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); - while(wasm.getMemValue(pSql, 'i8') - /* Maintenance reminder: ^^^^ _must_ be i8 or else we + while(pSql && wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we use i32 or similar then we read 4 bytes, read stuff around the NUL terminator, and get stuck in and endless loop at the end of the SQL, endlessly re-preparing an empty statement. */ ){ - wasm.setMemValue(ppStmt, 0, wasm.ptrIR); - wasm.setMemValue(pzTail, 0, wasm.ptrIR); - DB.checkRc(this, capi.sqlite3_prepare_v2( - this.pointer, pSql, sqlByteLen, ppStmt, pzTail + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail )); - const pStmt = wasm.getMemValue(ppStmt, wasm.ptrIR); - pSql = wasm.getMemValue(pzTail, wasm.ptrIR); + const pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); sqlByteLen = pSqlEnd - pSql; if(!pStmt) continue; if(Array.isArray(opt.saveSql)){ @@ -569,28 +624,30 @@ stmt.bind(bind); bind = null; } - if(stmt.columnCount && undefined!==rowMode){ + if(evalFirstResult && stmt.columnCount){ /* Only forward SELECT results for the FIRST query in the SQL which potentially has them. */ - while(stmt.step()){ + evalFirstResult = false; + if(Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + while(!!arg.cbArg && stmt.step()){ stmt._isLocked = true; const row = arg.cbArg(stmt); - if(callback) callback(row, stmt); if(resultRows) resultRows.push(row); + if(callback) callback.apply(opt,[row,stmt]); stmt._isLocked = false; } - rowMode = undefined; }else{ - // Do we need to while(stmt.step()){} here? stmt.step(); } stmt.finalize(); stmt = null; } - }catch(e){ - console.warn("DB.execMulti() is propagating exception",opt,e); + }/*catch(e){ + console.warn("DB.exec() is propagating exception",opt,e); throw e; - }finally{ + }*/finally{ if(stmt){ delete stmt._isLocked; stmt.finalize(); @@ -598,7 +655,7 @@ wasm.scopedAllocPop(stack); } return this; - }/*execMulti()*/, + }/*exec()*/, /** Creates a new scalar UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any @@ -680,8 +737,7 @@ let i, pVal, valType, arg; const tgt = []; for(i = 0; i < argc; ++i){ - pVal = capi.wasm.getMemValue(pArgv + (capi.wasm.ptrSizeof * i), - capi.wasm.ptrIR); + pVal = capi.wasm.getPtrValue(pArgv + (capi.wasm.ptrSizeof * i)); /** Curiously: despite ostensibly requiring 8-byte alignment, the pArgv array is parcelled into chunks of @@ -737,7 +793,7 @@ capi.sqlite3_result_null(pCx); break; }else if(util.isBindableTypedArray(val)){ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); capi.wasm.dealloc(pBlob); @@ -820,6 +876,49 @@ }, /** + Starts a transaction, calls the given callback, and then either + rolls back or commits the savepoint, depending on whether the + callback throws. The callback is passed this db object as its + only argument. On success, returns the result of the + callback. Throws on error. + + Note that transactions may not be nested, so this will throw if + it is called recursively. For nested transactions, use the + savepoint() method or manually manage SAVEPOINTs using exec(). + */ + transaction: function(callback){ + affirmDbOpen(this).exec("BEGIN"); + try { + const rc = callback(this); + this.exec("COMMIT"); + return rc; + }catch(e){ + this.exec("ROLLBACK"); + throw e; + } + }, + + /** + This works similarly to transaction() but uses sqlite3's SAVEPOINT + feature. This function starts a savepoint (with an unspecified name) + and calls the given callback function, passing it this db object. + If the callback returns, the savepoint is released (committed). If + the callback throws, the savepoint is rolled back. If it does not + throw, it returns the result of the callback. + */ + savepoint: function(callback){ + affirmDbOpen(this).exec("SAVEPOINT oo1"); + try { + const rc = callback(this); + this.exec("RELEASE oo1"); + return rc; + }catch(e){ + this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); + throw e; + } + }, + + /** This function currently does nothing and always throws. It WILL BE REMOVED pending other refactoring, to eliminate a hard dependency on Emscripten. This feature will be moved into a @@ -1028,7 +1127,7 @@ capi.wasm.scopedAllocPop(stack); } }else{ - const pBlob = capi.wasm.mallocFromTypedArray(val); + const pBlob = capi.wasm.allocFromTypedArray(val); try{ rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, capi.SQLITE_TRANSIENT); @@ -1042,7 +1141,7 @@ console.warn("Unsupported bind() argument type:",val); toss3("Unsupported bind() argument type: "+(typeof val)); } - if(rc) checkDbRc(stmt.db.pointer, rc); + if(rc) DB.checkRc(stmt.db.pointer, rc); return stmt; }; @@ -1059,6 +1158,7 @@ delete __stmtMap.get(this.db)[this.pointer]; capi.sqlite3_finalize(this.pointer); __ptrMap.delete(this); + delete this._mayGet; delete this.columnCount; delete this.parameterCount; delete this.db; @@ -1228,9 +1328,10 @@ return this; }, /** - Steps the statement one time. If the result indicates that - a row of data is available, true is returned. If no row of - data is available, false is returned. Throws on error. + Steps the statement one time. If the result indicates that a + row of data is available, a truthy value is returned. + If no row of data is available, a falsy + value is returned. Throws on error. */ step: function(){ affirmUnlocked(this, 'step()'); @@ -1242,8 +1343,51 @@ this._mayGet = false; console.warn("sqlite3_step() rc=",rc,"SQL =", capi.sqlite3_sql(this.pointer)); - checkDbRc(this.db.pointer, rc); - }; + DB.checkRc(this.db.pointer, rc); + } + }, + /** + Functions exactly like step() except that... + + 1) On success, it calls this.reset() and returns this object. + 2) On error, it throws and does not call reset(). + + This is intended to simplify constructs like: + + ``` + for(...) { + stmt.bind(...).stepReset(); + } + ``` + + Note that the reset() call makes it illegal to call this.get() + after the step. + */ + stepReset: function(){ + this.step(); + return this.reset(); + }, + /** + Functions like step() except that + it finalizes this statement immediately after stepping unless + the step cannot be performed because the statement is + locked. Throws on error, but any error other than the + statement-is-locked case will also trigger finalization of this + statement. + + On success, it returns true if the step indicated that a row of + data was available, else it returns false. + + This is intended to simplify use cases such as: + + ``` + aDb.prepare("insert in foo(a) values(?)").bind(123).stepFinalize(); + ``` + */ + stepFinalize: function(){ + const rc = this.step(); + this.finalize(); + return rc; }, /** Fetches the value from the given 0-based column index of @@ -1347,7 +1491,7 @@ default: toss3("Don't know how to translate", "type of result column #"+ndx+"."); } - abort("Not reached."); + toss3("Not reached."); }, /** Equivalent to get(ndx) but coerces the result to an integer. */ @@ -1434,5 +1578,20 @@ }, DB, Stmt - }/*SQLite3 object*/; -})(self); + }/*oo1 object*/; + + if( self.window===self && 0!==capi.sqlite3_vfs_find('kvvfs') ){ + /* In the main window thread, add a couple of convenience proxies + for localStorage and sessionStorage DBs... */ + let klass = sqlite3.oo1.LocalStorageDb = function(){ + dbCtorHelper.call(this, 'local', 'c', 'kvvfs'); + }; + klass.prototype = DB.prototype; + + klass = sqlite3.oo1.SessionStorageDb = function(){ + dbCtorHelper.call(this, 'session', 'c', 'kvvfs'); + }; + klass.prototype = DB.prototype; + } +}); + diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 4acab7770..693432b35 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -31,12 +31,13 @@ // FileSystemDirectoryHandle // FileSystemFileHandle // FileSystemFileHandle.prototype.createSyncAccessHandle -self.sqlite3.postInit.push(function(self, sqlite3){ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const warn = console.warn.bind(console), error = console.error.bind(console); - if(!self.importScripts || !self.FileSystemFileHandle - || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ - warn("OPFS not found or its sync API is not available in this environment."); + if(!self.importScripts || !self.FileSystemFileHandle){ + //|| !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ + // ^^^ sync API is not required with WASMFS/OPFS backend. + warn("OPFS is not available in this environment."); return; }else if(!sqlite3.capi.wasm.bigIntEnabled){ error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 60ed61477..17dcd4228 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -78,25 +78,110 @@ */ /** - This global symbol is is only a temporary measure: the JS-side - post-processing will remove that object from the global scope when - setup is complete. We require it there temporarily in order to glue - disparate parts together during the loading of the API (which spans - several components). + sqlite3ApiBootstrap() is the only global symbol exposed by this + API. It is intended to be called one time at the end of the API + amalgamation process, passed configuration details for the current + environment, and then optionally be removed from the global object + using `delete self.sqlite3ApiBootstrap`. - This function requires a configuration object intended to abstract + This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so - that it can be used without any _direct_ dependency on Emscripten. - (That said, OO API #1 requires, as of this writing, Emscripten's - virtual filesystem API. Baby steps.) + that it can be used without any _direct_ dependency on + Emscripten. The config object is only honored the first time this + is called. Subsequent calls ignore the argument and return the same + (configured) object which gets initialized by the first call. + + The config object properties include: + + - `Module`[^1]: Emscripten-style module object. Currently only required + by certain test code and is _not_ part of the public interface. + (TODO: rename this to EmscriptenModule to be more explicit.) + + - `exports`[^1]: the "exports" object for the current WASM + environment. In an Emscripten build, this should be set to + `Module['asm']`. + + - `memory`[^1]: optional WebAssembly.Memory object, defaulting to + `exports.memory`. In Emscripten environments this should be set + to `Module.wasmMemory` if the build uses `-sIMPORT_MEMORY`, or be + left undefined/falsy to default to `exports.memory` when using + WASM-exported memory. + + - `bigIntEnabled`: true if BigInt support is enabled. Defaults to + true if self.BigInt64Array is available, else false. Some APIs + will throw exceptions if called without BigInt support, as BigInt + is required for marshalling C-side int64 into and out of JS. + + - `allocExportName`: the name of the function, in `exports`, of the + `malloc(3)`-compatible routine for the WASM environment. Defaults + to `"malloc"`. + + - `deallocExportName`: the name of the function, in `exports`, of + the `free(3)`-compatible routine for the WASM + environment. Defaults to `"free"`. + + - `persistentDirName`[^1]: if the environment supports persistent storage, this + directory names the "mount point" for that directory. It must be prefixed + by `/` and may currently contain only a single directory-name part. Using + the root directory name is not supported by any current persistent backend. + + + [^1] = This property may optionally be a function, in which case this + function re-assigns it to the value returned from that function, + enabling delayed evaluation. + */ -self.sqlite3ApiBootstrap = function(config){ - 'use strict'; +'use strict'; +self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (sqlite3ApiBootstrap.defaultConfig || self.sqlite3ApiConfig) +){ + if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ + console.warn("sqlite3ApiBootstrap() called multiple times.", + "Config and external initializers are ignored on calls after the first."); + return sqlite3ApiBootstrap.sqlite3; + } + apiConfig = apiConfig || {}; + const config = Object.create(null); + { + const configDefaults = { + Module: undefined/*needed for some test code, not part of the public API*/, + exports: undefined, + memory: undefined, + bigIntEnabled: !!self.BigInt64Array, + allocExportName: 'malloc', + deallocExportName: 'free', + persistentDirName: '/persistent' + }; + Object.keys(configDefaults).forEach(function(k){ + config[k] = Object.getOwnPropertyDescriptor(apiConfig, k) + ? apiConfig[k] : configDefaults[k]; + }); + // Copy over any properties apiConfig defines but configDefaults does not... + Object.keys(apiConfig).forEach(function(k){ + if(!Object.getOwnPropertyDescriptor(config, k)){ + config[k] = apiConfig[k]; + } + }); + } + + [ + // If any of these config options are functions, replace them with + // the result of calling that function... + 'Module', 'exports', 'memory', 'persistentDirName' + ].forEach((k)=>{ + if('function' === typeof config[k]){ + config[k] = config[k](); + } + }); /** Throws a new Error, the message of which is the concatenation all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; + if(config.persistentDirName && !/^\/[^/]+$/.test(config.persistentDirName)){ + toss("config.persistentDirName must be falsy or in the form '/dir-name'."); + } + /** Returns true if n is a 32-bit (signed) integer, else false. This is used for determining when we need to switch to @@ -143,7 +228,18 @@ self.sqlite3ApiBootstrap = function(config){ }; const utf8Decoder = new TextDecoder('utf-8'); - const typedArrayToString = (str)=>utf8Decoder.decode(str); + + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const typedArrayToString = function(arrayBuffer, begin, end){ + return utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; /** An Error subclass specifically for reporting Wasm-level malloc() @@ -173,36 +269,6 @@ self.sqlite3ApiBootstrap = function(config){ */ const capi = { /** - An Error subclass which is thrown by this object's alloc() method - on OOM. - */ - WasmAllocError: WasmAllocError, - /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). - - That said, very few cases in the API can result in - client-defined functions propagating exceptions via the C-style - API. Most notably, this applies ot User-defined SQL Functions - (UDFs) registered via sqlite3_create_function_v2(). For that - specific case it is recommended that all UDF creation be - funneled through a utility function and that a wrapper function - be added around the UDF which catches any exception and sets - the error state to OOM. (The overall complexity of registering - UDFs essentially requires a helper for doing so!) - */ - alloc: undefined/*installed later*/, - /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). - */ - dealloc: undefined/*installed later*/, - /** When using sqlite3_open_v2() it is important to keep the following in mind: @@ -365,6 +431,33 @@ self.sqlite3ApiBootstrap = function(config){ || toss("API config object requires a WebAssembly.Memory object", "in either config.exports.memory (exported)", "or config.memory (imported)."), + + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + That said, very few cases in the API can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies ot User-defined SQL Functions + (UDFs) registered via sqlite3_create_function_v2(). For that + specific case it is recommended that all UDF creation be + funneled through a utility function and that a wrapper function + be added around the UDF which catches any exception and sets + the error state to OOM. (The overall complexity of registering + UDFs essentially requires a helper for doing so!) + */ + alloc: undefined/*installed later*/, + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + */ + dealloc: undefined/*installed later*/ + /* Many more wasm-related APIs get installed later on. */ }/*wasm*/ }/*capi*/; @@ -387,7 +480,7 @@ self.sqlite3ApiBootstrap = function(config){ Int8Array types and will throw if srcTypedArray is of any other type. */ - capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + capi.wasm.allocFromTypedArray = function(srcTypedArray){ affirmBindableTypedArray(srcTypedArray); const pRet = this.alloc(srcTypedArray.byteLength || 1); this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); @@ -400,11 +493,13 @@ self.sqlite3ApiBootstrap = function(config){ const f = capi.wasm.exports[key]; if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); } + capi.wasm.alloc = function(n){ const m = this.exports[keyAlloc](n); if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); return m; }.bind(capi.wasm) + capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); /** @@ -576,18 +671,125 @@ self.sqlite3ApiBootstrap = function(config){ ["sqlite3_total_changes64", "i64", ["sqlite3*"]] ]; + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + capi.wasm. + */ + capi.wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_vfs_unlink", "int", "string"] + ]; + + /** State for sqlite3_web_persistent_dir(). */ + let __persistentDir; + /** + An experiment. Do not use. + + If the wasm environment has a persistent storage directory, + its path is returned by this function. If it does not then + it returns "" (noting that "" is a falsy value). + + The first time this is called, this function inspects the current + environment to determine whether persistence filesystem support + is available and, if it is, enables it (if needed). + + TODOs and caveats: + + - If persistent storage is available at the root of the virtual + filesystem, this interface cannot currently distinguish that + from the lack of persistence. That case cannot currently (with + WASMFS/OPFS) happen, but it is conceivably possible in future + environments or non-browser runtimes (none of which are yet + supported targets). + */ + capi.sqlite3_web_persistent_dir = function(){ + if(undefined !== __persistentDir) return __persistentDir; + // If we have no OPFS, there is no persistent dir + const pdir = config.persistentDirName; + if(!pdir + || !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return __persistentDir = ""; + } + try{ + if(pdir && 0===capi.wasm.xCallWrapped( + 'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir + )){ + /** OPFS does not support locking and will trigger errors if + we try to lock. We don't _really_ want to + _unconditionally_ install a non-locking sqlite3 VFS as the + default, but we do so here for simplicy's sake for the + time being. That said: locking is a no-op on all of the + current WASM storage, so this isn't (currently) as bad as + it may initially seem. */ + const pVfs = sqlite3.capi.sqlite3_vfs_find("unix-none"); + if(pVfs){ + capi.sqlite3_vfs_register(pVfs,1); + console.warn("Installed 'unix-none' as the default sqlite3 VFS."); + } + return __persistentDir = pdir; + }else{ + return __persistentDir = ""; + } + }catch(e){ + // sqlite3_wasm_init_opfs() is not available + return __persistentDir = ""; + } + }; + + /** + Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a + non-empty string and the given name has that string as its + prefix, else returns false. + */ + capi.sqlite3_web_filename_is_persistent = function(name){ + const p = capi.sqlite3_web_persistent_dir(); + return (p && name) ? name.startsWith(p) : false; + }; + + if(0===capi.wasm.exports.sqlite3_vfs_find(0)){ + /* Assume that sqlite3_initialize() has not yet been called. + This will be the case in an SQLITE_OS_KV build. */ + capi.wasm.exports.sqlite3_initialize(); + } + /* The remainder of the API will be set up in later steps. */ - return { + const sqlite3 = { + WasmAllocError: WasmAllocError, capi, - postInit: [ - /* some pieces of the API may install functions into this array, - and each such function will be called, passed (self,sqlite3), - at the very end of the API load/init process, where self is - the current global object and sqlite3 is the object returned - from sqlite3ApiBootstrap(). This array will be removed at the - end of the API setup process. */], - /** Config is needed downstream for gluing pieces together. It - will be removed at the end of the API setup process. */ config }; + sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3)); + delete sqlite3ApiBootstrap.initializers; + sqlite3ApiBootstrap.sqlite3 = sqlite3; + return sqlite3; }/*sqlite3ApiBootstrap()*/; +/** + self.sqlite3ApiBootstrap.initializers is an internal detail used by + the various pieces of the sqlite3 API's amalgamation process. It + must not be modified by client code except when plugging such code + into the amalgamation process. + + Each component of the amalgamation is expected to append a function + to this array. When sqlite3ApiBootstrap() is called for the first + time, each such function will be called (in their appended order) + and passed the sqlite3 namespace object, into which they can install + their features (noting that most will also require that certain + features alread have been installed). At the end of that process, + this array is deleted. +*/ +self.sqlite3ApiBootstrap.initializers = []; +/** + Client code may assign sqlite3ApiBootstrap.defaultConfig an + object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging in + an environment-suitable configuration without having to define a new + global-scope symbol. +*/ +self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +/** Placeholder: gets installed by the first call to + self.sqlite3ApiBootstrap(). */ +self.sqlite3ApiBootstrap.sqlite3 = undefined; diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js deleted file mode 100644 index 95b27b21e..000000000 --- a/ext/wasm/api/sqlite3-api-worker.js +++ /dev/null @@ -1,420 +0,0 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file implements a Worker-based wrapper around SQLite3 OO API - #1. - - In order to permit this API to be loaded in worker threads without - automatically registering onmessage handlers, initializing the - worker API requires calling initWorkerAPI(). If this function - is called from a non-worker thread then it throws an exception. - - When initialized, it installs message listeners to receive messages - from the main thread and then it posts a message in the form: - - ``` - {type:'sqlite3-api',data:'worker-ready'} - ``` - - This file requires that the core C-style sqlite3 API and OO API #1 - have been loaded and that self.sqlite3 contains both, - as documented for those APIs. -*/ -self.sqlite3.initWorkerAPI = function(){ - 'use strict'; - /** - UNDER CONSTRUCTION - - We need an API which can proxy the DB API via a Worker message - interface. The primary quirky factor in such an API is that we - cannot pass callback functions between the window thread and a - worker thread, so we have to receive all db results via - asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who wants - to juggle six database handles at once. Do we add that complexity - or tell such users to write their own code using the provided - lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we collect - all results and bundle them back in a single message? The former - is, generically speaking, more memory-efficient but the latter - far easier to implement in this environment. The latter is - untennable for large data sets. Despite a web page hypothetically - being a relatively limited environment, there will always be - those users who feel that they should/need to be able to work - with multi-hundred-meg (or larger) blobs, and passing around - arrays of those may quickly exhaust the JS engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB.pointer (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB.pointer) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. - - - Revisit how virtual files are managed. We currently delete DBs - from the virtual filesystem when we close them, for the sake of - saving memory (the VFS lives in RAM). Supporting multiple DBs may - require that we give up that habit. Similarly, fully supporting - ATTACH, where a user can upload multiple DBs and ATTACH them, - also requires the that we manage the VFS entries better. - */ - const toss = (...args)=>{throw new Error(args.join(' '))}; - if('function' !== typeof importScripts){ - toss("Cannot initalize the sqlite3 worker API in the main thread."); - } - const self = this.self; - const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); - const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); - const DB = SQLite3.DB; - - /** - Returns the app-wide unique ID for the given db, creating one if - needed. - */ - const getDbId = function(db){ - let id = wState.idMap.get(db); - if(id) return id; - id = 'db#'+(++wState.idSeq)+'@'+db.pointer; - /** ^^^ can't simply use db.pointer b/c closing/opening may re-use - the same address, which could map pending messages to a wrong - instance. */ - wState.idMap.set(db, id); - return id; - }; - - /** - Helper for managing Worker-level state. - */ - const wState = { - defaultDb: undefined, - idSeq: 0, - idMap: new WeakMap, - open: function(arg){ - // TODO: if arg is a filename, look for a db in this.dbs with the - // same filename and close/reopen it (or just pass it back as is?). - if(!arg && this.defaultDb) return this.defaultDb; - //???if(this.defaultDb) this.defaultDb.close(); - let db; - db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); - this.dbs[getDbId(db)] = db; - if(!this.defaultDb) this.defaultDb = db; - return db; - }, - close: function(db,alsoUnlink){ - if(db){ - delete this.dbs[getDbId(db)]; - db.close(alsoUnlink); - if(db===this.defaultDb) this.defaultDb = undefined; - } - }, - post: function(type,data,xferList){ - if(xferList){ - self.postMessage({type, data},xferList); - xferList.length = 0; - }else{ - self.postMessage({type, data}); - } - }, - /** Map of DB IDs to DBs. */ - dbs: Object.create(null), - getDb: function(id,require=true){ - return this.dbs[id] - || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); - } - }; - - /** Throws if the given db is falsy or not opened. */ - const affirmDbOpen = function(db = wState.defaultDb){ - return (db && db.pointer) ? db : toss("DB is not opened."); - }; - - /** Extract dbId from the given message payload. */ - const getMsgDb = function(msgData,affirmExists=true){ - const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; - return affirmExists ? affirmDbOpen(db) : db; - }; - - const getDefaultDbId = function(){ - return wState.defaultDb && getDbId(wState.defaultDb); - }; - - /** - A level of "organizational abstraction" for the Worker - API. Each method in this object must map directly to a Worker - message type key. The onmessage() dispatcher attempts to - dispatch all inbound messages to a method of this object, - passing it the event.data part of the inbound event object. All - methods must return a plain Object containing any response - state, which the dispatcher may amend. All methods must throw - on error. - */ - const wMsgHandler = { - xfer: [/*Temp holder for "transferable" postMessage() state.*/], - /** - Proxy for DB.exec() which expects a single argument of type - string (SQL to execute) or an options object in the form - expected by exec(). The notable differences from exec() - include: - - - The default value for options.rowMode is 'array' because - the normal default cannot cross the window/Worker boundary. - - - A function-type options.callback property cannot cross - the window/Worker boundary, so is not useful here. If - options.callback is a string then it is assumed to be a - message type key, in which case a callback function will be - applied which posts each row result via: - - postMessage({type: thatKeyType, data: theRow}) - - And, at the end of the result set (whether or not any - result rows were produced), it will post an identical - message with data:null to alert the caller than the result - set is completed. - - The callback proxy must not recurse into this interface, or - results are undefined. (It hypothetically cannot recurse - because an exec() call will be tying up the Worker thread, - causing any recursion attempt to wait until the first - exec() is completed.) - - The response is the input options object (or a synthesized - one if passed only a string), noting that - options.resultRows and options.columnNames may be populated - by the call to exec(). - - This opens/creates the Worker's db if needed. - */ - exec: function(ev){ - const opt = ( - 'string'===typeof ev.data - ) ? {sql: ev.data} : (ev.data || Object.create(null)); - if(undefined===opt.rowMode){ - /* Since the default rowMode of 'stmt' is not useful - for the Worker interface, we'll default to - something else. */ - opt.rowMode = 'array'; - }else if('stmt'===opt.rowMode){ - toss("Invalid rowMode for exec(): stmt mode", - "does not work in the Worker API."); - } - const db = getMsgDb(ev); - if(opt.callback || Array.isArray(opt.resultRows)){ - // Part of a copy-avoidance optimization for blobs - db._blobXfer = this.xfer; - } - const callbackMsgType = opt.callback; - if('string' === typeof callbackMsgType){ - /* Treat this as a worker message type and post each - row as a message of that type. */ - const that = this; - opt.callback = - (row)=>wState.post(callbackMsgType,row,this.xfer); - } - try { - db.exec(opt); - if(opt.callback instanceof Function){ - opt.callback = callbackMsgType; - wState.post(callbackMsgType, null); - } - }/*catch(e){ - console.warn("Worker is propagating:",e);throw e; - }*/finally{ - delete db._blobXfer; - if(opt.callback){ - opt.callback = callbackMsgType; - } - } - return opt; - }/*exec()*/, - /** - TO(re)DO, once we can abstract away access to the - JS environment's virtual filesystem. Currently this - always throws. - - Response is (should be) an object: - - { - buffer: Uint8Array (db file contents), - filename: the current db filename, - mimetype: 'application/x-sqlite3' - } - - TODO is to determine how/whether this feature can support - exports of ":memory:" and "" (temp file) DBs. The latter is - ostensibly easy because the file is (potentially) on disk, but - the former does not have a structure which maps directly to a - db file image. - */ - export: function(ev){ - toss("export() requires reimplementing for portability reasons."); - /**const db = getMsgDb(ev); - const response = { - buffer: db.exportBinaryImage(), - filename: db.filename, - mimetype: 'application/x-sqlite3' - }; - this.xfer.push(response.buffer.buffer); - return response;**/ - }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input), - - id: an opaque ID value intended for future distinction - between multiple db handles. Messages including a specific - ID will use the DB for that ID. - - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.simulateError){ - toss("Throwing because of open.simulateError flag."); - } - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return { - filename: db.filename, - dbId: getDbId(db) - }; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - { - filename: db filename _if_ the db is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const db = getMsgDb(ev,false); - const response = { - filename: db && db.filename - }; - if(db){ - wState.close(db, !!((ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data)); - } - return response; - }, - toss: function(ev){ - toss("Testing worker exception"); - } - }/*wMsgHandler*/; - - /** - UNDER CONSTRUCTION! - - A subset of the DB API is accessible via Worker messages in the - form: - - { type: apiCommand, - dbId: optional DB ID value (else uses a default db handle) - data: apiArguments - } - - As a rule, these commands respond with a postMessage() of their - own in the same form, but will, if needed, transform the `data` - member to an object and may add state to it. The responses - always have an object-format `data` part. If the inbound `data` - is an object which has a `messageId` property, that property is - always mirrored in the result object, for use in client-side - dispatching of these asynchronous results. Exceptions thrown - during processing result in an `error`-type event with a - payload in the form: - - { - message: error string, - errorClass: class name of the error type, - dbId: DB handle ID, - input: ev.data, - [messageId: if set in the inbound message] - } - - The individual APIs are documented in the wMsgHandler object. - */ - self.onmessage = function(ev){ - ev = ev.data; - let response, dbId = ev.dbId, evType = ev.type; - const arrivalTime = performance.now(); - try { - if(wMsgHandler.hasOwnProperty(evType) && - wMsgHandler[evType] instanceof Function){ - response = wMsgHandler[evType](ev); - }else{ - toss("Unknown db worker message type:",ev.type); - } - }catch(err){ - evType = 'error'; - response = { - message: err.message, - errorClass: err.name, - input: ev - }; - if(err.stack){ - response.stack = ('string'===typeof err.stack) - ? err.stack.split('\n') : err.stack; - } - if(0) console.warn("Worker is propagating an exception to main thread.", - "Reporting it _here_ for the stack trace:",err,response); - } - if(!response.messageId && ev.data - && 'object'===typeof ev.data && ev.data.messageId){ - response.messageId = ev.data.messageId; - } - if(!dbId){ - dbId = response.dbId/*from 'open' cmd*/ - || getDefaultDbId(); - } - if(!response.dbId) response.dbId = dbId; - // Timing info is primarily for use in testing this API. It's not part of - // the public API. arrivalTime = when the worker got the message. - response.workerReceivedTime = arrivalTime; - response.workerRespondTime = performance.now(); - response.departureTime = ev.departureTime; - wState.post(evType, response, wMsgHandler.xfer); - }; - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); -}.bind({self, sqlite3: self.sqlite3}); diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js new file mode 100644 index 000000000..00359413b --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -0,0 +1,621 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + sqlite3.initWorker1API() implements a Worker-based wrapper around + SQLite3 OO API #1, colloquially known as "Worker API #1". + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorker1API(). If this function is + called from a non-worker thread then it throws an exception. It + must only be called once per Worker. + + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: + + ``` + {type:'sqlite3-api', result:'worker1-ready'} + ``` + + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. + + Note that the worker-based interface can be slightly quirky because + of its async nature. In particular, any number of messages may be posted + to the worker before it starts handling any of them. If, e.g., an + "open" operation fails, any subsequent messages will fail. The + Promise-based wrapper for this API (`sqlite3-worker1-promiser.js`) + is more comfortable to use in that regard. + + The documentation for the input and output worker messages for + this API follows... + + ==================================================================== + Common message format... + + Each message posted to the worker has an operation-independent + envelope and operation-dependent arguments: + + ``` + { + type: string, // one of: 'open', 'close', 'exec', 'config-get' + + messageId: OPTIONAL arbitrary value. The worker will copy it as-is + into response messages to assist in client-side dispatching. + + dbId: a db identifier string (returned by 'open') which tells the + operation which database instance to work on. If not provided, the + first-opened db is used. This is an "opaque" value, with no + inherently useful syntax or information. Its value is subject to + change with any given build of this API and cannot be used as a + basis for anything useful beyond its one intended purpose. + + args: ...operation-dependent arguments... + + // the framework may add other properties for testing or debugging + // purposes. + + } + ``` + + Response messages, posted back to the main thread, look like: + + ``` + { + type: string. Same as above except for error responses, which have the type + 'error', + + messageId: same value, if any, provided by the inbound message + + dbId: the id of the db which was operated on, if any, as returned + by the corresponding 'open' operation. + + result: ...operation-dependent result... + + } + ``` + + ==================================================================== + Error responses + + Errors are reported messages in an operation-independent format: + + ``` + { + type: 'error', + + messageId: ...as above..., + + dbId: ...as above... + + result: { + + operation: type of the triggering operation: 'open', 'close', ... + + message: ...error message text... + + errorClass: string. The ErrorClass.name property from the thrown exception. + + input: the message object which triggered the error. + + stack: _if available_, a stack trace array. + + } + + } + ``` + + + ==================================================================== + "config-get" + + This operation fetches the serializable parts of the sqlite3 API + configuration. + + Message format: + + ``` + { + type: "config-get", + messageId: ...as above..., + args: currently ignored and may be elided. + } + ``` + + Response: + + ``` + { + type: 'config', + messageId: ...as above..., + result: { + + persistentDirName: path prefix, if any, of persistent storage. + An empty string denotes that no persistent storage is available. + + bigIntEnabled: bool. True if BigInt support is enabled. + + persistenceEnabled: true if persistent storage is enabled in the + current environment. Only files stored under persistentDirName + will persist, however. + + } + } + ``` + + + ==================================================================== + "open" a database + + Message format: + + ``` + { + type: "open", + messageId: ...as above..., + args:{ + + filename [=":memory:" or "" (unspecified)]: the db filename. + See the sqlite3.oo1.DB constructor for peculiarities and transformations, + + persistent [=false]: if true and filename is not one of ("", + ":memory:"), prepend sqlite3.capi.sqlite3_web_persistent_dir() + to the given filename so that it is stored in persistent storage + _if_ the environment supports it. If persistent storage is not + supported, the filename is used as-is. + + } + } + ``` + + Response: + + ``` + { + type: 'open', + messageId: ...as above..., + result: { + filename: db filename, possibly differing from the input. + + dbId: an opaque ID value which must be passed in the message + envelope to other calls in this API to tell them which db to + use. If it is not provided to future calls, they will default to + operating on the first-opened db. This property is, for API + consistency's sake, also part of the contaning message envelope. + Only the `open` operation includes it in the `result` property. + + persistent: true if the given filename resides in the + known-persistent storage, else false. This determination is + independent of the `persistent` input argument. + } + } + ``` + + ==================================================================== + "close" a database + + Message format: + + ``` + { + type: "close", + messageId: ...as above... + dbId: ...as above... + args: OPTIONAL: { + + unlink: if truthy, the associated db will be unlinked (removed) + from the virtual filesystems. Failure to unlink is silently + ignored. + + } + } + ``` + + If the dbId does not refer to an opened ID, this is a no-op. The + inability to close a db (because it's not opened) or delete its + file does not trigger an error. + + Response: + + ``` + { + type: 'close', + messageId: ...as above..., + result: { + + filename: filename of closed db, or undefined if no db was closed + + } + } + ``` + + ==================================================================== + "exec" SQL + + All SQL execution is processed through the exec operation. It offers + most of the features of the oo1.DB.exec() method, with a few limitations + imposed by the state having to cross thread boundaries. + + Message format: + + ``` + { + type: "exec", + messageId: ...as above... + dbId: ...as above... + args: string (SQL) or {... see below ...} + } + ``` + + Response: + + ``` + { + type: 'exec', + messageId: ...as above..., + dbId: ...as above... + result: { + input arguments, possibly modified. See below. + } + } + ``` + + The arguments are in the same form accepted by oo1.DB.exec(), with + the exceptions noted below. + + A function-type args.callback property cannot cross + the window/Worker boundary, so is not useful here. If + args.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, + rowNumber: 1-based-#, + row: theRow, + columnNames: anArray + }) + + And, at the end of the result set (whether or not any result rows + were produced), it will post an identical message with + (row=undefined, rowNumber=null) to alert the caller than the result + set is completed. Note that a row value of `null` is a legal row + result for certain arg.rowMode values. + + (Design note: we don't use (row=undefined, rowNumber=undefined) to + indicate end-of-results because fetching those would be + indistinguishable from fetching from an empty object unless the + client used hasOwnProperty() (or similar) to distinguish "missing + property" from "property with the undefined value". Similarly, + `null` is a legal value for `row` in some case , whereas the db + layer won't emit a result value of `undefined`.) + + The callback proxy must not recurse into this interface. An exec() + call will type up the Worker thread, causing any recursion attempt + to wait until the first exec() is completed. + + The response is the input options object (or a synthesized one if + passed only a string), noting that options.resultRows and + options.columnNames may be populated by the call to db.exec(). + +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +sqlite3.initWorker1API = function(){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("Cannot initalize the sqlite3 worker API in the main thread."); + } + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const SQLite3 = sqlite3.oo1 || toss("Missing this.sqlite3.oo1 OO API."); + const DB = SQLite3.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Internal helper for managing Worker-level state. + */ + const wState = { + /** First-opened db is the default for future operations when no + dbId is provided by the client. */ + defaultDb: undefined, + /** Sequence number of dbId generation. */ + idSeq: 0, + /** Map of DB instances to dbId. */ + idMap: new WeakMap, + /** Temp holder for "transferable" postMessage() state. */ + xfer: [], + open: function(opt){ + const db = new DB(opt.filename); + this.dbs[getDbId(db)] = db; + if(!this.defaultDb) this.defaultDb = db; + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + const filename = db.fileName(); + db.close(); + if(db===this.defaultDb) this.defaultDb = undefined; + if(alsoUnlink && filename){ + sqlite3.capi.wasm.sqlite3_wasm_vfs_unlink(filename); + } + } + }, + /** + Posts the given worker message value. If xferList is provided, + it must be an array, in which case a copy of it passed as + postMessage()'s second argument and xferList.length is set to + 0. + */ + post: function(msg,xferList){ + if(xferList && xferList.length){ + self.postMessage( msg, Array.from(xferList) ); + xferList.length = 0; + }else{ + self.postMessage(msg); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + /** Fetch the DB for the given id. Throw if require=true and the + id is not valid, else return the db or undefined. */ + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened. */ + const affirmDbOpen = function(db = wState.defaultDb){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.defaultDb && getDbId(wState.defaultDb); + }; + + /** + A level of "organizational abstraction" for the Worker + API. Each method in this object must map directly to a Worker + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any result + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + open: function(ev){ + const oargs = Object.create(null), args = (ev.args || Object.create(null)); + if(args.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + const rc = Object.create(null); + const pDir = sqlite3.capi.sqlite3_web_persistent_dir(); + if(!args.filename || ':memory:'===args.filename){ + oargs.filename = args.filename || ''; + }else if(pDir){ + oargs.filename = pDir + ('/'===args.filename[0] ? args.filename : ('/'+args.filename)); + }else{ + oargs.filename = args.filename; + } + const db = wState.open(oargs); + rc.filename = db.filename; + rc.persistent = !!pDir && db.filename.startsWith(pDir); + rc.dbId = getDbId(db); + return rc; + }, + + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + wState.close(db, ((ev.args && 'object'===typeof ev.args) + ? !!ev.args.unlink : false)); + } + return response; + }, + + exec: function(ev){ + const rc = ( + 'string'===typeof ev.args + ) ? {sql: ev.args} : (ev.args || Object.create(null)); + if('stmt'===rc.rowMode){ + toss("Invalid rowMode for 'exec': stmt mode", + "does not work in the Worker API."); + }else if(!rc.sql){ + toss("'exec' requires input SQL."); + } + const db = getMsgDb(ev); + if(rc.callback || Array.isArray(rc.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = wState.xfer; + } + const theCallback = rc.callback; + let rowNumber = 0; + const hadColNames = !!rc.columnNames; + if('string' === typeof theCallback){ + if(!hadColNames) rc.columnNames = []; + /* Treat this as a worker message type and post each + row as a message of that type. */ + rc.callback = function(row,stmt){ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: ++rowNumber, + row: row + }, wState.xfer); + } + } + try { + db.exec(rc); + if(rc.callback instanceof Function){ + rc.callback = theCallback; + /* Post a sentinel message to tell the client that the end + of the result set has been reached (possibly with zero + rows). */ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: null /*null to distinguish from "property not set"*/, + row: undefined /*undefined because null is a legal row value + for some rowType values, but undefined is not*/ + }); + } + }finally{ + delete db._blobXfer; + if(rc.callback) rc.callback = theCallback; + } + return rc; + }/*exec()*/, + + 'config-get': function(){ + const rc = Object.create(null), src = sqlite3.config; + [ + 'persistentDirName', 'bigIntEnabled' + ].forEach(function(k){ + if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; + }); + rc.persistenceEnabled = !!sqlite3.capi.sqlite3_web_persistent_dir(); + return rc; + }, + + /** + TO(RE)DO, once we can abstract away access to the + JS environment's virtual filesystem. Currently this + always throws. + + Response is (should be) an object: + + { + buffer: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + + TODO is to determine how/whether this feature can support + exports of ":memory:" and "" (temp file) DBs. The latter is + ostensibly easy because the file is (potentially) on disk, but + the former does not have a structure which maps directly to a + db file image. We can VACUUM INTO a :memory:/temp db into a + file for that purpose, though. + */ + export: function(ev){ + toss("export() requires reimplementing for portability reasons."); + /** + We need to reimplement this to use the Emscripten FS + interface. That part used to be in the OO#1 API but that + dependency was removed from that level of the API. + */ + /**const db = getMsgDb(ev); + const response = { + buffer: db.exportBinaryImage(), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + wState.xfer.push(response.buffer.buffer); + return response;**/ + }/*export()*/, + + toss: function(ev){ + toss("Testing worker exception"); + } + }/*wMsgHandler*/; + + self.onmessage = function(ev){ + ev = ev.data; + let result, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + result = wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + result = { + operation: ev.type, + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + result.stack = ('string'===typeof err.stack) + ? err.stack.split(/\n\s*/) : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,result); + } + if(!dbId){ + dbId = result.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + wState.post({ + type: evType, + dbId: dbId, + messageId: ev.messageId, + workerReceivedTime: arrivalTime, + workerRespondTime: performance.now(), + departureTime: ev.departureTime, + // TODO: move the timing bits into... + //timing:{ + // departure: ev.departureTime, + // workerReceived: arrivalTime, + // workerResponse: performance.now(); + //}, + result: result + }, wState.xfer); + }; + self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); +}.bind({self, sqlite3}); +}); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 6a81da3e5..2a505f19a 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1,6 +1,40 @@ +/* +** This file requires access to sqlite3.c static state in order to +** implement certain WASM-specific features. Unlike the rest of +** sqlite3.c, this file requires compiling with -std=c99 (or +** equivalent, or a later C version) because it makes use of features +** not available in C89. +*/ #include "sqlite3.c" /* +** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not +** Emscripten-specific. It explicitly includes marked functions for +** export into the target wasm file without requiring explicit listing +** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list +** (or equivalent in other build platforms). Any function with neither +** this attribute nor which is listed as an explicit export will not +** be exported from the wasm file (but may still be used internally +** within the wasm file). +** +** The functions in this file (sqlite3-wasm.c) which require exporting +** are marked with this flag. They may also be added to any explicit +** build-time export list but need not be. All of these APIs are +** intended for use only within the project's own JS/WASM code, and +** not by client code, so an argument can be made for reducing their +** visibility by not including them in any build-time export lists. +** +** 2022-09-11: it's not yet _proven_ that this approach works in +** non-Emscripten builds. If not, such builds will need to export +** those using the --export=... wasm-ld flag (or equivalent). As of +** this writing we are tied to Emscripten for various reasons +** and cannot test the library with other build environments. +*/ +#define WASM_KEEP __attribute__((used,visibility("default"))) +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) + +/* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** @@ -14,8 +48,8 @@ ** ** Returns err_code. */ -int sqlite3_wasm_db_error(sqlite3*db, int err_code, - const char *zMsg){ +WASM_KEEP +int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ if(0!=zMsg){ const int nMsg = sqlite3Strlen30(zMsg); sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); @@ -40,6 +74,7 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, ** buffer is not large enough for the generated JSON. In debug builds ** that will trigger an assert(). */ +WASM_KEEP const char * sqlite3_wasm_enum_json(void){ static char strBuf[1024 * 8] = {0} /* where the JSON goes */; int n = 0, childCount = 0, structCount = 0 @@ -411,3 +446,82 @@ const char * sqlite3_wasm_enum_json(void){ #undef outf #undef lenCheck } + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function invokes the xDelete method of the default VFS, +** passing on the given filename. If zName is NULL, no default VFS is +** found, or it has no xDelete method, SQLITE_MISUSE is returned, else +** the result of the xDelete() call is returned. +*/ +WASM_KEEP +int sqlite3_wasm_vfs_unlink(const char * zName){ + int rc = SQLITE_MISUSE /* ??? */; + sqlite3_vfs * const pVfs = sqlite3_vfs_find(0); + if( zName && pVfs && pVfs->xDelete ){ + rc = pVfs->xDelete(pVfs, zName, 1); + } + return rc; +} + +#if defined(__EMSCRIPTEN__) && defined(SQLITE_WASM_OPFS) +#include <emscripten/wasmfs.h> +#include <emscripten/console.h> + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings, specifically +** only when building with Emscripten's WASMFS support. +** +** This function should only be called if the JS side detects the +** existence of the Origin-Private FileSystem (OPFS) APIs in the +** client. The first time it is called, this function instantiates a +** WASMFS backend impl for OPFS. On success, subsequent calls are +** no-ops. +** +** This function may be passed a "mount point" name, which must have a +** leading "/" and is currently restricted to a single path component, +** e.g. "/foo" is legal but "/foo/" and "/foo/bar" are not. If it is +** NULL or empty, it defaults to "/persistent". +** +** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend +** object fails, SQLITE_IOERR if mkdir() of the zMountPoint dir in +** the virtual FS fails. In builds compiled without SQLITE_WASM_OPFS +** defined, SQLITE_NOTFOUND is returned without side effects. +*/ +WASM_KEEP +int sqlite3_wasm_init_opfs(const char *zMountPoint){ + static backend_t pOpfs = 0; + if( !zMountPoint || !*zMountPoint ) zMountPoint = "/persistent"; + if( !pOpfs ){ + pOpfs = wasmfs_create_opfs_backend(); + if( pOpfs ){ + emscripten_console_log("Created WASMFS OPFS backend."); + } + } + /** It's not enough to instantiate the backend. We have to create a + mountpoint in the VFS and attach the backend to it. */ + if( pOpfs && 0!=access(zMountPoint, F_OK) ){ + /* mkdir() simply hangs when called from fiddle app. Cause is + not yet determined but the hypothesis is an init-order + issue. */ + /* Note that this check and is not robust but it will + hypothetically suffice for the transient wasm-based virtual + filesystem we're currently running in. */ + const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs); + emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc); + if(rc) return SQLITE_IOERR; + } + return pOpfs ? 0 : SQLITE_NOMEM; +} +#else +WASM_KEEP +int sqlite3_wasm_init_opfs(void){ + return SQLITE_NOTFOUND; +} +#endif /* __EMSCRIPTEN__ && SQLITE_WASM_OPFS */ + + +#undef WASM_KEEP diff --git a/ext/wasm/batch-runner.html b/ext/wasm/batch-runner.html new file mode 100644 index 000000000..38f38070c --- /dev/null +++ b/ext/wasm/batch-runner.html @@ -0,0 +1,63 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>sqlite3-api batch SQL runner</title> + </head> + <body> + <header id='titlebar'><span>sqlite3-api batch SQL runner</span></header> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <p> + This page is for batch-running extracts from the output + of <tt>speedtest1 --script</tt>, as well as other standalone SQL + scripts. + </p> + <p id='warn-list' class='warning'>ACHTUNG: this file requires a generated input list + file. Run "make batch" from this directory to generate it. + </p> + <p id='warn-opfs' class='warning'>WARNING: if the WASMFS/OPFS layer crashes, this page may + become completely unresponsive and need to be closed and + reloaded to recover. + </p> + <hr> + <div> + <select class='disable-during-eval' id='sql-select'> + <option disabled selected>Populated via script code</option> + </select> + <button class='disable-during-eval' id='sql-run'>Run selected SQL</button> + <button class='disable-during-eval' id='sql-run-next'>Run next...</button> + <button class='disable-during-eval' id='sql-run-remaining'>Run all remaining...</button> + <button class='disable-during-eval' id='export-metrics'>Export metrics (WIP)</button> + <button class='disable-during-eval' id='db-reset'>Reset db</button> + <button id='output-clear'>Clear output</button> + <span class='input-wrapper'> + <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input> + <label for='cb-reverse-log-order'>Reverse log order</label> + </span> + </div> + <hr> + <div id='reverse-log-notice' class='hidden'>(Log output is in reverse order, newest first!)</div> + <div id='test-output'></div> + + <script src="sqlite3.js"></script> + <script src="common/SqliteTestUtil.js"></script> + <script src="batch-runner.js"></script> + </body> +</html> diff --git a/ext/wasm/batch-runner.js b/ext/wasm/batch-runner.js new file mode 100644 index 000000000..437424b48 --- /dev/null +++ b/ext/wasm/batch-runner.js @@ -0,0 +1,405 @@ +/* + 2022-08-29 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic batch SQL runner for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const warn = console.warn.bind(console); + + const App = { + e: { + output: document.querySelector('#test-output'), + selSql: document.querySelector('#sql-select'), + btnRun: document.querySelector('#sql-run'), + btnRunNext: document.querySelector('#sql-run-next'), + btnRunRemaining: document.querySelector('#sql-run-remaining'), + btnExportMetrics: document.querySelector('#export-metrics'), + btnClear: document.querySelector('#output-clear'), + btnReset: document.querySelector('#db-reset'), + cbReverseLog: document.querySelector('#cb-reverse-log-order') + }, + cache:{}, + metrics:{ + /** + Map of sql-file to timing metrics. We currently only store + the most recent run of each file, but we really should store + all runs so that we can average out certain values which vary + significantly across runs. e.g. a mandelbrot-generating query + will have a wide range of runtimes when run 10 times in a + row. + */ + }, + log: console.log.bind(console), + warn: console.warn.bind(console), + cls: function(){this.e.output.innerHTML = ''}, + logHtml2: function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + this.e.output.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + }, + logHtml: function(...args){ + console.log(...args); + if(1) this.logHtml2('', ...args); + }, + logErr: function(...args){ + console.error(...args); + if(1) this.logHtml2('error', ...args); + }, + + openDb: function(fn, unlinkFirst=true){ + if(this.db && this.db.ptr){ + toss("Already have an opened db."); + } + const capi = this.sqlite3.capi, wasm = capi.wasm; + const stack = wasm.scopedAllocPush(); + let pDb = 0; + try{ + if(unlinkFirst && fn && ':memory:'!==fn){ + capi.wasm.sqlite3_wasm_vfs_unlink(fn); + } + const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + const ppDb = wasm.scopedAllocPtr(); + const rc = capi.sqlite3_open_v2(fn, ppDb, oFlags, null); + pDb = wasm.getPtrValue(ppDb) + if(rc){ + if(pDb) capi.sqlite3_close_v2(pDb); + toss("sqlite3_open_v2() failed with code",rc); + } + }finally{ + wasm.scopedAllocPop(stack); + } + this.db = Object.create(null); + this.db.filename = fn; + this.db.ptr = pDb; + this.logHtml("Opened db:",fn); + return this.db.ptr; + }, + + closeDb: function(unlink=false){ + if(this.db && this.db.ptr){ + this.sqlite3.capi.sqlite3_close_v2(this.db.ptr); + this.logHtml("Closed db",this.db.filename); + if(unlink) capi.wasm.sqlite3_wasm_vfs_unlink(this.db.filename); + this.db.ptr = this.db.filename = undefined; + } + }, + + /** + Loads batch-runner.list and populates the selection list from + it. Returns a promise which resolves to nothing in particular + when it completes. Only intended to be run once at the start + of the app. + */ + loadSqlList: async function(){ + const sel = this.e.selSql; + sel.innerHTML = ''; + this.blockControls(true); + const infile = 'batch-runner.list'; + this.logHtml("Loading list of SQL files:", infile); + let txt; + try{ + const r = await fetch(infile); + if(404 === r.status){ + toss("Missing file '"+infile+"'."); + } + if(!r.ok) toss("Loading",infile,"failed:",r.statusText); + txt = await r.text(); + const warning = document.querySelector('#warn-list'); + if(warning) warning.remove(); + }catch(e){ + this.logErr(e.message); + throw e; + }finally{ + this.blockControls(false); + } + const list = txt.split(/\n+/); + let opt; + if(0){ + opt = document.createElement('option'); + opt.innerText = "Select file to evaluate..."; + opt.value = ''; + opt.disabled = true; + opt.selected = true; + sel.appendChild(opt); + } + list.forEach(function(fn){ + if(!fn) return; + opt = document.createElement('option'); + opt.value = fn; + opt.innerText = fn.split('/').pop(); + sel.appendChild(opt); + }); + this.logHtml("Loaded",infile); + }, + + /** Fetch ./fn and return its contents as a Uint8Array. */ + fetchFile: async function(fn, cacheIt=false){ + if(cacheIt && this.cache[fn]) return this.cache[fn]; + this.logHtml("Fetching",fn,"..."); + let sql; + try { + const r = await fetch(fn); + if(!r.ok) toss("Fetch failed:",r.statusText); + sql = new Uint8Array(await r.arrayBuffer()); + }catch(e){ + this.logErr(e.message); + throw e; + } + this.logHtml("Fetched",sql.length,"bytes from",fn); + if(cacheIt) this.cache[fn] = sql; + return sql; + }/*fetchFile()*/, + + /** Throws if the given sqlite3 result code is not 0. */ + checkRc: function(rc){ + if(this.db.ptr && rc){ + toss("Prepare failed:",this.sqlite3.capi.sqlite3_errmsg(this.db.ptr)); + } + }, + + /** Disable or enable certain UI controls. */ + blockControls: function(disable){ + document.querySelectorAll('.disable-during-eval').forEach((e)=>e.disabled = disable); + }, + + /** + Converts this.metrics() to a form which is suitable for easy conversion to + CSV. It returns an array of arrays. The first sub-array is the column names. + The 2nd and subsequent are the values, one per test file (only the most recent + metrics are kept for any given file). + */ + metricsToArrays: function(){ + const rc = []; + Object.keys(this.metrics).sort().forEach((k)=>{ + const m = this.metrics[k]; + delete m.evalFileStart; + delete m.evalFileEnd; + const mk = Object.keys(m).sort(); + if(!rc.length){ + rc.push(['file', ...mk]); + } + const row = [k.split('/').pop()/*remove dir prefix from filename*/]; + rc.push(row); + mk.forEach((kk)=>row.push(m[kk])); + }); + return rc; + }, + + metricsToBlob: function(colSeparator='\t'){ + const ar = [], ma = this.metricsToArrays(); + if(!ma.length){ + this.logErr("Metrics are empty. Run something."); + return; + } + ma.forEach(function(row){ + ar.push(row.join(colSeparator),'\n'); + }); + return new Blob(ar); + }, + + downloadMetrics: function(){ + const b = this.metricsToBlob(); + if(!b) return; + const url = URL.createObjectURL(b); + const a = document.createElement('a'); + a.href = url; + a.download = 'batch-runner-js-'+((new Date().getTime()/1000) | 0)+'.csv'; + this.logHtml("Triggering download of",a.download); + document.body.appendChild(a); + a.click(); + setTimeout(()=>{ + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, 500); + }, + + /** + Fetch file fn and eval it as an SQL blob. This is an async + operation and returns a Promise which resolves to this + object on success. + */ + evalFile: async function(fn){ + const sql = await this.fetchFile(fn); + const banner = "========================================"; + this.logHtml(banner, + "Running",fn,'('+sql.length,'bytes)...'); + const capi = this.sqlite3.capi, wasm = capi.wasm; + let pStmt = 0, pSqlBegin; + const stack = wasm.scopedAllocPush(); + const metrics = this.metrics[fn] = Object.create(null); + metrics.prepTotal = metrics.stepTotal = 0; + metrics.stmtCount = 0; + metrics.malloc = 0; + metrics.strcpy = 0; + this.blockControls(true); + if(this.gotErr){ + this.logErr("Cannot run ["+fn+"]: error cleanup is pending."); + return; + } + // Run this async so that the UI can be updated for the above header... + const ff = function(resolve, reject){ + metrics.evalFileStart = performance.now(); + try { + let t; + let sqlByteLen = sql.byteLength; + const [ppStmt, pzTail] = wasm.scopedAllocPtr(2); + t = performance.now(); + pSqlBegin = wasm.alloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed"); + metrics.malloc = performance.now() - t; + metrics.byteLength = sqlByteLen; + let pSql = pSqlBegin; + const pSqlEnd = pSqlBegin + sqlByteLen; + t = performance.now(); + wasm.heap8().set(sql, pSql); + wasm.setMemValue(pSql + sqlByteLen, 0); + metrics.strcpy = performance.now() - t; + let breaker = 0; + while(pSql && wasm.getMemValue(pSql,'i8')){ + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + t = performance.now(); + let rc = capi.sqlite3_prepare_v3( + this.db.ptr, pSql, sqlByteLen, 0, ppStmt, pzTail + ); + metrics.prepTotal += performance.now() - t; + this.checkRc(rc); + pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue/*empty statement*/; + ++metrics.stmtCount; + t = performance.now(); + rc = capi.sqlite3_step(pStmt); + capi.sqlite3_finalize(pStmt); + pStmt = 0; + metrics.stepTotal += performance.now() - t; + switch(rc){ + case capi.SQLITE_ROW: + case capi.SQLITE_DONE: break; + default: this.checkRc(rc); toss("Not reached."); + } + } + }catch(e){ + if(pStmt) capi.sqlite3_finalize(pStmt); + this.gotErr = e; + //throw e; + reject(e); + return; + }finally{ + wasm.dealloc(pSqlBegin); + wasm.scopedAllocPop(stack); + this.blockControls(false); + } + metrics.evalFileEnd = performance.now(); + metrics.evalTimeTotal = (metrics.evalFileEnd - metrics.evalFileStart); + this.logHtml("Metrics:");//,JSON.stringify(metrics, undefined, ' ')); + this.logHtml("prepare() count:",metrics.stmtCount); + this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms", + "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())"); + this.logHtml("Time in step():",metrics.stepTotal,"ms", + "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())"); + this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms"); + this.logHtml("Overhead (time - prep - step):", + (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); + this.logHtml(banner,"End of",fn); + resolve(this); + }.bind(this); + let p; + if(1){ + p = new Promise(function(res,rej){ + setTimeout(()=>ff(res, rej), 50)/*give UI a chance to output the "running" banner*/; + }); + }else{ + p = new Promise(ff); + } + return p.catch((e)=>this.logErr("Error via evalFile("+fn+"):",e.message)); + }/*evalFile()*/, + + run: function(sqlite3){ + delete this.run; + this.sqlite3 = sqlite3; + const capi = sqlite3.capi, wasm = capi.wasm; + this.logHtml("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + this.logHtml("WASM heap size =",wasm.heap8().length); + this.loadSqlList(); + const pDir = capi.sqlite3_web_persistent_dir(); + const dbFile = pDir ? pDir+"/speedtest.db" : ":memory:"; + if(!pDir){ + document.querySelector('#warn-opfs').remove(); + } + this.openDb(dbFile, !!pDir); + const who = this; + const eReverseLogNotice = document.querySelector('#reverse-log-notice'); + if(this.e.cbReverseLog.checked){ + eReverseLogNotice.classList.remove('hidden'); + this.e.output.classList.add('reverse'); + } + this.e.cbReverseLog.addEventListener('change', function(){ + if(this.checked){ + who.e.output.classList.add('reverse'); + eReverseLogNotice.classList.remove('hidden'); + }else{ + who.e.output.classList.remove('reverse'); + eReverseLogNotice.classList.add('hidden'); + } + }, false); + this.e.btnClear.addEventListener('click', ()=>this.cls(), false); + this.e.btnRun.addEventListener('click', function(){ + if(!who.e.selSql.value) return; + who.evalFile(who.e.selSql.value); + }, false); + this.e.btnRunNext.addEventListener('click', function(){ + ++who.e.selSql.selectedIndex; + if(!who.e.selSql.value) return; + who.evalFile(who.e.selSql.value); + }, false); + this.e.btnReset.addEventListener('click', function(){ + const fn = who.db.filename; + if(fn){ + who.closeDb(true); + who.openDb(fn,true); + } + }, false); + this.e.btnExportMetrics.addEventListener('click', function(){ + who.logHtml2('warning',"Triggering download of metrics CSV. Check your downloads folder."); + who.downloadMetrics(); + //const m = who.metricsToArrays(); + //console.log("Metrics:",who.metrics, m); + }); + this.e.btnRunRemaining.addEventListener('click', async function(){ + let v = who.e.selSql.value; + const timeStart = performance.now(); + while(v){ + await who.evalFile(v); + if(who.gotError){ + who.logErr("Error handling script",v,":",who.gotError.message); + break; + } + ++who.e.selSql.selectedIndex; + v = who.e.selSql.value; + } + const timeTotal = performance.now() - timeStart; + who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))"); + }, false); + }/*run()*/ + }/*App*/; + + self.sqlite3TestModule.initSqlite3().then(function(theEmccModule){ + self._MODULE = theEmccModule /* this is only to facilitate testing from the console */; + App.run(theEmccModule.sqlite3); + }); +})(); diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js index c7c99240e..779f271fb 100644 --- a/ext/wasm/common/SqliteTestUtil.js +++ b/ext/wasm/common/SqliteTestUtil.js @@ -113,6 +113,46 @@ ++this.counter; if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); return this; + }, + + /** + Parses window.location.search-style string into an object + containing key/value pairs of URL arguments (already + urldecoded). The object is created using Object.create(null), + so contains only parsed-out properties and has no prototype + (and thus no inherited properties). + + If the str argument is not passed (arguments.length==0) then + window.location.search.substring(1) is used by default. If + neither str is passed in nor window exists then false is returned. + + On success it returns an Object containing the key/value pairs + parsed from the string. Keys which have no value are treated + has having the boolean true value. + + Pedantic licensing note: this code has appeared in other source + trees, but was originally written by the same person who pasted + it into those trees. + */ + processUrlArgs: function(str) { + if( 0 === arguments.length ) { + if( ('undefined' === typeof window) || + !window.location || + !window.location.search ) return false; + else str = (''+window.location.search).substring(1); + } + if( ! str ) return false; + str = (''+str).split(/#/,2)[0]; // remove #... to avoid it being added as part of the last value. + const args = Object.create(null); + const sp = str.split(/&+/); + const rx = /^([^=]+)(=(.+))?/; + var i, m; + for( i in sp ) { + m = rx.exec( sp[i] ); + if( ! m ) continue; + args[decodeURIComponent(m[1])] = (m[3] ? decodeURIComponent(m[3]) : true); + } + return args; } }; @@ -122,6 +162,11 @@ sqlite3InitModule() factory function. */ self.sqlite3TestModule = { + /** + Array of functions to call after Emscripten has initialized the + wasm module. Each gets passed the Emscripten module object + (which is _this_ object). + */ postRun: [ /* function(theModule){...} */ ], @@ -135,10 +180,10 @@ console.error.apply(console, Array.prototype.slice.call(arguments)); }, /** - Called by the module init bits to report loading - progress. It gets passed an empty argument when loading is - done (after onRuntimeInitialized() and any this.postRun - callbacks have been run). + Called by the Emscripten module init bits to report loading + progress. It gets passed an empty argument when loading is done + (after onRuntimeInitialized() and any this.postRun callbacks + have been run). */ setStatus: function f(text){ if(!f.last){ @@ -168,6 +213,30 @@ } f.ui.status.classList.add('hidden'); } + }, + /** + Config options used by the Emscripten-dependent initialization + which happens via this.initSqlite3(). This object gets + (indirectly) passed to sqlite3ApiBootstrap() to configure the + sqlite3 API. + */ + sqlite3ApiConfig: { + persistentDirName: "/persistent" + }, + /** + Intended to be called by apps which need to call the + Emscripten-installed sqlite3InitModule() routine. This function + temporarily installs this.sqlite3ApiConfig into the self + object, calls it sqlite3InitModule(), and removes + self.sqlite3ApiConfig after initialization is done. Returns the + promise from sqlite3InitModule(), and the next then() handler + will get the Emscripten module object as its argument. That + module has the sqlite3's main namespace object installed as its + `sqlite3` property. + */ + initSqlite3: function(){ + self.sqlite3ApiConfig = this.sqlite3ApiConfig; + return self.sqlite3InitModule(this).finally(()=>delete self.sqlite3ApiConfig); } }; })(self/*window or worker*/); diff --git a/ext/wasm/common/testing.css b/ext/wasm/common/testing.css index 09c570f48..e112fd0a8 100644 --- a/ext/wasm/common/testing.css +++ b/ext/wasm/common/testing.css @@ -1,3 +1,8 @@ +body { + display: flex; + flex-direction: column; + flex-wrap: wrap; +} textarea { font-family: monospace; } @@ -29,4 +34,17 @@ span.labeled-input { color: red; background-color: yellow; } -#test-output { font-family: monospace } +.warning { color: firebrick; } +.input-wrapper { white-space: nowrap; } +#test-output { + border: 1px inset; + padding: 0.25em; + /*max-height: 30em;*/ + overflow: auto; + white-space: break-spaces; + display: flex; flex-direction: column; + font-family: monospace; +} +#test-output.reverse { + flex-direction: column-reverse; +} diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 5a1d425ca..42f602d00 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -212,9 +212,10 @@ self.WhWasmUtilInstaller = function(target){ that will certainly change. */ const ptrIR = target.pointerIR || 'i32'; - const ptrSizeof = ('i32'===ptrIR ? 4 - : ('i64'===ptrIR - ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); + const ptrSizeof = target.ptrSizeof = + ('i32'===ptrIR ? 4 + : ('i64'===ptrIR + ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); /** Stores various cached state. */ const cache = Object.create(null); /** Previously-recorded size of cache.memory.buffer, noted so that @@ -326,7 +327,7 @@ self.WhWasmUtilInstaller = function(target){ if(c.HEAP64) return unsigned ? c.HEAP64U : c.HEAP64; break; default: - if(this.bigIntEnabled){ + if(target.bigIntEnabled){ if(n===self['BigUint64Array']) return c.HEAP64U; else if(n===self['BigInt64Array']) return c.HEAP64; break; @@ -334,7 +335,7 @@ self.WhWasmUtilInstaller = function(target){ } toss("Invalid heapForSize() size: expecting 8, 16, 32,", "or (if BigInt is enabled) 64."); - }.bind(target); + }; /** Returns the WASM-exported "indirect function table." @@ -346,16 +347,16 @@ self.WhWasmUtilInstaller = function(target){ - Use `__indirect_function_table` as the import name for the table, which is what LLVM does. */ - }.bind(target); + }; /** Given a function pointer, returns the WASM function table entry if found, else returns a falsy value. */ target.functionEntry = function(fptr){ - const ft = this.functionTable(); + const ft = target.functionTable(); return fptr < ft.length ? ft.get(fptr) : undefined; - }.bind(target); + }; /** Creates a WASM function which wraps the given JS function and @@ -504,7 +505,7 @@ self.WhWasmUtilInstaller = function(target){ https://github.com/emscripten-core/emscripten/issues/17323 */ target.installFunction = function f(func, sig){ - const ft = this.functionTable(); + const ft = target.functionTable(); const oldLen = ft.length; let ptr; while(cache.freeFuncIndexes.length){ @@ -532,13 +533,13 @@ self.WhWasmUtilInstaller = function(target){ } // It's not a WASM-exported function, so compile one... try { - ft.set(ptr, this.jsFuncToWasm(func, sig)); + ft.set(ptr, target.jsFuncToWasm(func, sig)); }catch(e){ if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); throw e; } return ptr; - }.bind(target); + }; /** Requires a pointer value previously returned from @@ -551,12 +552,12 @@ self.WhWasmUtilInstaller = function(target){ */ target.uninstallFunction = function(ptr){ const fi = cache.freeFuncIndexes; - const ft = this.functionTable(); + const ft = target.functionTable(); fi.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); return rc; - }.bind(target); + }; /** Given a WASM heap memory address and a data type name in the form @@ -614,14 +615,14 @@ self.WhWasmUtilInstaller = function(target){ case 'i16': return c.HEAP16[ptr>>1]; case 'i32': return c.HEAP32[ptr>>2]; case 'i64': - if(this.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); + if(target.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); break; case 'float': case 'f32': return c.HEAP32F[ptr>>2]; case 'double': case 'f64': return Number(c.HEAP64F[ptr>>3]); default: break; } toss('Invalid type for getMemValue():',type); - }.bind(target); + }; /** The counterpart of getMemValue(), this sets a numeric value at @@ -654,6 +655,15 @@ self.WhWasmUtilInstaller = function(target){ toss('Invalid type for setMemValue(): ' + type); }; + + /** Convenience form of getMemValue() intended for fetching + pointer-to-pointer values. */ + target.getPtrValue = (ptr)=>target.getMemValue(ptr, ptrIR); + + /** Convenience form of setMemValue() intended for setting + pointer-to-pointer values. */ + target.setPtrValue = (ptr, value)=>target.setMemValue(ptr, value, ptrIR); + /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. @@ -669,6 +679,18 @@ self.WhWasmUtilInstaller = function(target){ return pos - ptr; }; + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const __utf8Decode = function(arrayBuffer, begin, end){ + return cache.utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; + /** Expects ptr to be a pointer into the WASM heap memory which refers to a NUL-terminated C-style string encoded as UTF-8. This @@ -677,13 +699,9 @@ self.WhWasmUtilInstaller = function(target){ ptr is falsy, `null` is returned. */ target.cstringToJs = function(ptr){ - const n = this.cstrlen(ptr); - if(null===n) return n; - return n - ? cache.utf8Decoder.decode( - new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) - ) : ""; - }.bind(target); + const n = target.cstrlen(ptr); + return n ? __utf8Decode(heapWrappers().HEAP8U, ptr, ptr+n) : (null===n ? n : ""); + }; /** Given a JS string, this function returns its UTF-8 length in @@ -811,16 +829,16 @@ self.WhWasmUtilInstaller = function(target){ */ target.cstrncpy = function(tgtPtr, srcPtr, n){ if(!tgtPtr || !srcPtr) toss("cstrncpy() does not accept NULL strings."); - if(n<0) n = this.cstrlen(strPtr)+1; + if(n<0) n = target.cstrlen(strPtr)+1; else if(!(n>0)) return 0; - const heap = this.heap8u(); + const heap = target.heap8u(); let i = 0, ch; for(; i < n && (ch = heap[srcPtr+i]); ++i){ heap[tgtPtr+i] = ch; } if(i<n) heap[tgtPtr + i++] = 0; return i; - }.bind(target); + }; /** For the given JS string, returns a Uint8Array of its contents @@ -865,13 +883,13 @@ self.WhWasmUtilInstaller = function(target){ }; const __allocCStr = function(jstr, returnWithLength, allocator, funcName){ - __affirmAlloc(this, funcName); + __affirmAlloc(target, funcName); if('string'!==typeof jstr) return null; - const n = this.jstrlen(jstr), + const n = target.jstrlen(jstr), ptr = allocator(n+1); - this.jstrcpy(jstr, this.heap8u(), ptr, n+1, true); + target.jstrcpy(jstr, target.heap8u(), ptr, n+1, true); return returnWithLength ? [ptr, n] : ptr; - }.bind(target); + }; /** Uses target.alloc() to allocate enough memory for jstrlen(jstr)+1 @@ -924,11 +942,11 @@ self.WhWasmUtilInstaller = function(target){ alloc levels are currently active. */ target.scopedAllocPush = function(){ - __affirmAlloc(this, 'scopedAllocPush'); + __affirmAlloc(target, 'scopedAllocPush'); const a = []; cache.scopedAlloc.push(a); return a; - }.bind(target); + }; /** Cleans up all allocations made using scopedAlloc() in the context @@ -953,15 +971,15 @@ self.WhWasmUtilInstaller = function(target){ trivial code that may be a non-issue. */ target.scopedAllocPop = function(state){ - __affirmAlloc(this, 'scopedAllocPop'); + __affirmAlloc(target, 'scopedAllocPop'); const n = arguments.length ? cache.scopedAlloc.indexOf(state) : cache.scopedAlloc.length-1; if(n<0) toss("Invalid state object for scopedAllocPop()."); if(0===arguments.length) state = cache.scopedAlloc[n]; cache.scopedAlloc.splice(n,1); - for(let p; (p = state.pop()); ) this.dealloc(p); - }.bind(target); + for(let p; (p = state.pop()); ) target.dealloc(p); + }; /** Allocates n bytes of memory using this.alloc() and records that @@ -983,10 +1001,10 @@ self.WhWasmUtilInstaller = function(target){ if(!cache.scopedAlloc.length){ toss("No scopedAllocPush() scope is active."); } - const p = this.alloc(n); + const p = target.alloc(n); cache.scopedAlloc[cache.scopedAlloc.length-1].push(p); return p; - }.bind(target); + }; Object.defineProperty(target.scopedAlloc, 'level', { configurable: false, enumerable: false, @@ -1005,6 +1023,29 @@ self.WhWasmUtilInstaller = function(target){ target.scopedAlloc, 'scopedAllocCString()'); /** + Creates an array, using scopedAlloc(), suitable for passing to a + C-level main() routine. The input is a collection with a length + property and a forEach() method. A block of memory list.length + entries long is allocated and each pointer-sized block of that + memory is populated with a scopedAllocCString() conversion of the + (''+value) of each element. Returns a pointer to the start of the + list, suitable for passing as the 2nd argument to a C-style + main() function. + + Throws if list.length is falsy or scopedAllocPush() is not active. + */ + target.scopedAllocMainArgv = function(list){ + if(!list.length) toss("Cannot allocate empty array."); + const pList = target.scopedAlloc(list.length * target.ptrSizeof); + let i = 0; + list.forEach((e)=>{ + target.setPtrValue(pList + (target.ptrSizeof * i++), + target.scopedAllocCString(""+e)); + }); + return pList; + }; + + /** Wraps function call func() in a scopedAllocPush() and scopedAllocPop() block, such that all calls to scopedAlloc() and friends from within that call will have their memory freed @@ -1013,15 +1054,15 @@ self.WhWasmUtilInstaller = function(target){ result of calling func(). */ target.scopedAllocCall = function(func){ - this.scopedAllocPush(); - try{ return func() } finally{ this.scopedAllocPop() } - }.bind(target); + target.scopedAllocPush(); + try{ return func() } finally{ target.scopedAllocPop() } + }; /** Internal impl for allocPtr() and scopedAllocPtr(). */ const __allocPtr = function(howMany, method){ - __affirmAlloc(this, method); - let m = this[method](howMany * ptrSizeof); - this.setMemValue(m, 0, ptrIR) + __affirmAlloc(target, method); + let m = target[method](howMany * ptrSizeof); + target.setMemValue(m, 0, ptrIR) if(1===howMany){ return m; } @@ -1029,10 +1070,10 @@ self.WhWasmUtilInstaller = function(target){ for(let i = 1; i < howMany; ++i){ m += ptrSizeof; a[i] = m; - this.setMemValue(m, 0, ptrIR); + target.setMemValue(m, 0, ptrIR); } return a; - }.bind(target); + }; /** Allocates a single chunk of memory capable of holding `howMany` @@ -1070,11 +1111,11 @@ self.WhWasmUtilInstaller = function(target){ /** Looks up a WASM-exported function named fname from - target.exports. If found, it is called, passed all remaining + target.exports. If found, it is called, passed all remaining arguments, and its return value is returned to xCall's caller. If not found, an exception is thrown. This function does no - conversion of argument or return types, but see xWrap() - and xCallWrapped() for variants which do. + conversion of argument or return types, but see xWrap() and + xCallWrapped() for variants which do. As a special case, if passed only 1 argument after the name and that argument in an Array, that array's entries become the @@ -1082,7 +1123,7 @@ self.WhWasmUtilInstaller = function(target){ not legal to pass an Array object to a WASM function.) */ target.xCall = function(fname, ...args){ - const f = this.xGet(fname); + const f = target.xGet(fname); if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); if(f.length!==args.length) __argcMismatch(fname,f.length) /* This is arguably over-pedantic but we want to help clients keep @@ -1090,7 +1131,7 @@ self.WhWasmUtilInstaller = function(target){ return (2===arguments.length && Array.isArray(arguments[1])) ? f.apply(null, arguments[1]) : f.apply(null, args); - }.bind(target); + }; /** State for use with xWrap() @@ -1164,19 +1205,19 @@ self.WhWasmUtilInstaller = function(target){ */ xcv.arg['func-ptr'] = function(v){ if(!(v instanceof Function)) return xcv.arg[ptrIR]; - const f = this.jsFuncToWasm(v, WHAT_SIGNATURE); - }.bind(target); + const f = target.jsFuncToWasm(v, WHAT_SIGNATURE); + }; } - const __xArgAdapter = + const __xArgAdapterCheck = (t)=>xcv.arg[t] || toss("Argument adapter not found:",t); - const __xResultAdapter = + const __xResultAdapterCheck = (t)=>xcv.result[t] || toss("Result adapter not found:",t); - cache.xWrap.convertArg = (t,v)=>__xArgAdapter(t)(v); + cache.xWrap.convertArg = (t,v)=>__xArgAdapterCheck(t)(v); cache.xWrap.convertResult = - (t,v)=>(null===t ? v : (t ? __xResultAdapter(t)(v) : undefined)); + (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); /** Creates a wrapper for the WASM-exported function fname. Uses @@ -1310,34 +1351,32 @@ self.WhWasmUtilInstaller = function(target){ if(3===arguments.length && Array.isArray(arguments[2])){ argTypes = arguments[2]; } - const xf = this.xGet(fname); - if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length) + const xf = target.xGet(fname); + if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length); if((null===resultType) && 0===xf.length){ /* Func taking no args with an as-is return. We don't need a wrapper. */ return xf; } /*Verify the arg type conversions are valid...*/; - if(undefined!==resultType && null!==resultType) __xResultAdapter(resultType); - argTypes.forEach(__xArgAdapter) + if(undefined!==resultType && null!==resultType) __xResultAdapterCheck(resultType); + argTypes.forEach(__xArgAdapterCheck); if(0===xf.length){ // No args to convert, so we can create a simpler wrapper... - return function(){ - return (arguments.length - ? __argcMismatch(fname, xf.length) - : cache.xWrap.convertResult(resultType, xf.call(null))); - }; + return (...args)=>(args.length + ? __argcMismatch(fname, xf.length) + : cache.xWrap.convertResult(resultType, xf.call(null))); } return function(...args){ if(args.length!==xf.length) __argcMismatch(fname, xf.length); - const scope = this.scopedAllocPush(); + const scope = target.scopedAllocPush(); try{ const rc = xf.apply(null,args.map((v,i)=>cache.xWrap.convertArg(argTypes[i], v))); return cache.xWrap.convertResult(resultType, rc); }finally{ - this.scopedAllocPop(scope); + target.scopedAllocPop(scope); } - }.bind(this); - }.bind(target)/*xWrap()*/; + }; + }/*xWrap()*/; /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ @@ -1441,9 +1480,9 @@ self.WhWasmUtilInstaller = function(target){ */ target.xCallWrapped = function(fname, resultType, argTypes, ...args){ if(Array.isArray(arguments[3])) args = arguments[3]; - return this.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); - }.bind(target); - + return target.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); + }; + return target; }; @@ -1522,10 +1561,11 @@ self.WhWasmUtilInstaller.yawl = function(config){ || toss("Missing 'memory' object!"); } if(!tgt.alloc && arg.instance.exports.malloc){ + const exports = arg.instance.exports; tgt.alloc = function(n){ - return this(n) || toss("Allocation of",n,"bytes failed."); - }.bind(arg.instance.exports.malloc); - tgt.dealloc = function(m){this(m)}.bind(arg.instance.exports.free); + return exports.malloc(n) || toss("Allocation of",n,"bytes failed."); + }; + tgt.dealloc = function(m){exports.free(m)}; } wui(tgt); } diff --git a/ext/wasm/demo-oo1.html b/ext/wasm/demo-oo1.html new file mode 100644 index 000000000..9b6e8cbfa --- /dev/null +++ b/ext/wasm/demo-oo1.html @@ -0,0 +1,34 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>sqlite3-api OO #1 Demo</title> + </head> + <body> + <header id='titlebar'><span>sqlite3-api OO #1 Demo</span></header> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <div>Most stuff on this page happens in the dev console.</div> + <hr> + <div id='test-output'></div> + <script src="sqlite3.js"></script> + <script src="common/SqliteTestUtil.js"></script> + <script src="demo-oo1.js"></script> + </body> +</html> diff --git a/ext/wasm/demo-oo1.js b/ext/wasm/demo-oo1.js new file mode 100644 index 000000000..4564fe0dd --- /dev/null +++ b/ext/wasm/demo-oo1.js @@ -0,0 +1,232 @@ +/* + 2022-08-16 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic demonstration of the SQLite3 OO API #1, shorn of assertions + and the like to improve readability. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const debug = console.debug.bind(console), + log = console.log.bind(console), + warn = console.warn.bind(console), + error = console.error.bind(console); + + const demo1 = function(sqlite3){ + const capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + + const dbName = ( + 0 ? "" : capi.sqlite3_web_persistent_dir() + )+"/mydb.sqlite3" + if(0 && capi.sqlite3_web_persistent_dir()){ + capi.wasm.sqlite3_wasm_vfs_unlink(dbName); + } + const db = new oo.DB(dbName); + log("db =",db.filename); + /** + Never(!) rely on garbage collection to clean up DBs and + (especially) statements. Always wrap their lifetimes in + try/finally construct... + */ + try { + log("Create a table..."); + db.exec("CREATE TABLE IF NOT EXISTS t(a,b)"); + //Equivalent: + db.exec({ + sql:"CREATE TABLE IF NOT EXISTS t(a,b)" + // ... numerous other options ... + }); + // SQL can be either a string or a byte array + + log("Insert some data using exec()..."); + let i; + for( i = 1; i <= 5; ++i ){ + db.exec({ + sql: "insert into t(a,b) values (?,?)", + // bind by parameter index... + bind: [i, i*2] + }); + db.exec({ + sql: "insert into t(a,b) values ($a,$b)", + // bind by parameter name... + bind: {$a: i * 3, $b: i * 4} + }); + } + + log("Insert using a prepared statement..."); + let q = db.prepare("insert into t(a,b) values(?,?)"); + try { + for( i = 100; i < 103; ++i ){ + q.bind( [i, i*2] ).step(); + q.reset(); + } + // Equivalent... + for( i = 103; i <= 105; ++i ){ + q.bind(1, i).bind(2, i*2).stepReset(); + } + }finally{ + q.finalize(); + } + + log("Query data with exec() using rowMode 'array'..."); + db.exec({ + sql: "select a from t order by a limit 3", + rowMode: 'array', // 'array', 'object', or 'stmt' (default) + callback: function(row){ + log("row ",++this.counter,"=",row); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode 'object'..."); + db.exec({ + sql: "select a as aa, b as bb from t order by aa limit 3", + rowMode: 'object', + callback: function(row){ + log("row ",++this.counter,"=",row); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode 'stmt'..."); + db.exec({ + sql: "select a from t order by a limit 3", + rowMode: 'stmt', // stmt === the default + callback: function(row){ + log("row ",++this.counter,"get(0) =",row.get(0)); + }.bind({counter: 0}) + }); + + log("Query data with exec() using rowMode INTEGER (result column index)..."); + db.exec({ + sql: "select a, b from t order by a limit 3", + rowMode: 1, // === result column 1 + callback: function(row){ + log("row ",++this.counter,"b =",row); + }.bind({counter: 0}) + }); + + log("Query data with exec() without a callback..."); + let resultRows = []; + db.exec({ + sql: "select a, b from t order by a limit 3", + rowMode: 'object', + resultRows: resultRows + }); + log("Result rows:",resultRows); + + log("Create a scalar UDF..."); + db.createFunction({ + name: 'twice', + callback: function(arg){ // note the call arg count + return arg + arg; + } + }); + log("Run scalar UDF and collect result column names..."); + let columnNames = []; + db.exec({ + sql: "select a, twice(a), twice(''||a) from t order by a desc limit 3", + columnNames: columnNames, + rowMode: 'stmt', + callback: function(row){ + log("a =",row.get(0), "twice(a) =", row.get(1), + "twice(''||a) =",row.get(2)); + } + }); + log("Result column names:",columnNames); + + if(0){ + warn("UDF will throw because of incorrect arg count..."); + db.exec("select twice(1,2,3)"); + } + + try { + db.transaction( function(D) { + D.exec("delete from t"); + log("In transaction: count(*) from t =",db.selectValue("select count(*) from t")); + throw new sqlite3.SQLite3Error("Demonstrating transaction() rollback"); + }); + }catch(e){ + if(e instanceof sqlite3.SQLite3Error){ + log("Got expected exception from db.transaction():",e.message); + log("count(*) from t =",db.selectValue("select count(*) from t")); + }else{ + throw e; + } + } + + try { + db.savepoint( function(D) { + D.exec("delete from t"); + log("In savepoint: count(*) from t =",db.selectValue("select count(*) from t")); + D.savepoint(function(DD){ + const rows = []; + D.exec({ + sql: ["insert into t(a,b) values(99,100);", + "select count(*) from t"], + rowMode: 0, + resultRows: rows + }); + log("In nested savepoint. Row count =",rows[0]); + throw new sqlite3.SQLite3Error("Demonstrating nested savepoint() rollback"); + }) + }); + }catch(e){ + if(e instanceof sqlite3.SQLite3Error){ + log("Got expected exception from nested db.savepoint():",e.message); + log("count(*) from t =",db.selectValue("select count(*) from t")); + }else{ + throw e; + } + } + + }finally{ + db.close(); + } + + /** + Misc. DB features: + + - get change count (total or statement-local, 32- or 64-bit) + - get its file name + - selectValue() takes SQL and returns first column of first row. + + Misc. Stmt features: + + - Various forms of bind() + - clearBindings() + - reset() + - Various forms of step() + - Variants of get() for explicit type treatment/conversion, + e.g. getInt(), getFloat(), getBlob(), getJSON() + - getColumnName(ndx), getColumnNames() + - getParamIndex(name) + */ + }/*demo1()*/; + + const runDemos = function(Module){ + //log("Module.sqlite3",Module); + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi; + log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + log("sqlite3 namespace:",sqlite3); + try { + demo1(sqlite3); + }catch(e){ + error("Exception:",e.message); + throw e; + } + }; + + //self.sqlite3TestModule.sqlite3ApiConfig.persistentDirName = "/hi"; + self.sqlite3TestModule.initSqlite3().then(runDemos); +})(); diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index ca562323c..5bc139175 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -89,213 +89,214 @@ */ "use strict"; (function(){ + /** + Posts a message in the form {type,data} unless passed more than 2 + args, in which case it posts {type, data:[arg1...argN]}. + */ + const wMsg = function(type,data){ + postMessage({ + type, + data: arguments.length<3 + ? data + : Array.prototype.slice.call(arguments,1) + }); + }; + + const stdout = function(){wMsg('stdout', Array.prototype.slice.call(arguments));}; + const stderr = function(){wMsg('stderr', Array.prototype.slice.call(arguments));}; + + self.onerror = function(/*message, source, lineno, colno, error*/) { + const err = arguments[4]; + if(err && 'ExitStatus'==err.name){ + /* This is relevant for the sqlite3 shell binding but not the + lower-level binding. */ + fiddleModule.isDead = true; + stderr("FATAL ERROR:", err.message); + stderr("Restarting the app requires reloading the page."); + wMsg('error', err); + } + console.error(err); + fiddleModule.setStatus('Exception thrown, see JavaScript console: '+err); + }; + + const Sqlite3Shell = { + /** Returns the name of the currently-opened db. */ + dbFilename: function f(){ + if(!f._) f._ = fiddleModule.cwrap('fiddle_db_filename', "string", ['string']); + return f._(); + }, /** - Posts a message in the form {type,data} unless passed more than 2 - args, in which case it posts {type, data:[arg1...argN]}. + Runs the given text through the shell as if it had been typed + in by a user. Fires a working/start event before it starts and + working/end event when it finishes. */ - const wMsg = function(type,data){ - postMessage({ - type, - data: arguments.length<3 - ? data - : Array.prototype.slice.call(arguments,1) - }); - }; - - const stdout = function(){wMsg('stdout', Array.prototype.slice.call(arguments));}; - const stderr = function(){wMsg('stderr', Array.prototype.slice.call(arguments));}; - - self.onerror = function(/*message, source, lineno, colno, error*/) { - const err = arguments[4]; - if(err && 'ExitStatus'==err.name){ - /* This is relevant for the sqlite3 shell binding but not the - lower-level binding. */ - fiddleModule.isDead = true; - stderr("FATAL ERROR:", err.message); - stderr("Restarting the app requires reloading the page."); - wMsg('error', err); + exec: function f(sql){ + if(!f._) f._ = fiddleModule.cwrap('fiddle_exec', null, ['string']); + if(fiddleModule.isDead){ + stderr("shell module has exit()ed. Cannot run SQL."); + return; + } + wMsg('working','start'); + try { + if(f._running){ + stderr('Cannot run multiple commands concurrently.'); + }else{ + f._running = true; + f._(sql); } - console.error(err); - fiddleModule.setStatus('Exception thrown, see JavaScript console: '+err); - }; - - const Sqlite3Shell = { - /** Returns the name of the currently-opened db. */ - dbFilename: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_db_filename', "string", ['string']); - return f._(); - }, - /** - Runs the given text through the shell as if it had been typed - in by a user. Fires a working/start event before it starts and - working/end event when it finishes. - */ - exec: function f(sql){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_exec', null, ['string']); - if(fiddleModule.isDead){ - stderr("shell module has exit()ed. Cannot run SQL."); - return; - } - wMsg('working','start'); - try { - if(f._running){ - stderr('Cannot run multiple commands concurrently.'); - }else{ - f._running = true; - f._(sql); - } - } finally { - delete f._running; - wMsg('working','end'); - } - }, - resetDb: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_reset_db', null); - stdout("Resetting database."); - f._(); - stdout("Reset",this.dbFilename()); - }, - /* Interrupt can't work: this Worker is tied up working, so won't get the - interrupt event which would be needed to perform the interrupt. */ - interrupt: function f(){ - if(!f._) f._ = fiddleModule.cwrap('fiddle_interrupt', null); - stdout("Requesting interrupt."); - f._(); - } - }; - - self.onmessage = function f(ev){ - ev = ev.data; - if(!f.cache){ - f.cache = { - prevFilename: null - }; - } - //console.debug("worker: onmessage.data",ev); - switch(ev.type){ - case 'shellExec': Sqlite3Shell.exec(ev.data); return; - case 'db-reset': Sqlite3Shell.resetDb(); return; - case 'interrupt': Sqlite3Shell.interrupt(); return; - /** Triggers the export of the current db. Fires an - event in the form: - - {type:'db-export', - data:{ - filename: name of db, - buffer: contents of the db file (Uint8Array), - error: on error, a message string and no buffer property. - } - } - */ - case 'db-export': { - const fn = Sqlite3Shell.dbFilename(); - stdout("Exporting",fn+"."); - const fn2 = fn ? fn.split(/[/\\]/).pop() : null; - try{ - if(!fn2) throw new Error("DB appears to be closed."); - wMsg('db-export',{ - filename: fn2, - buffer: fiddleModule.FS.readFile(fn, {encoding:"binary"}) - }); - }catch(e){ - /* Post a failure message so that UI elements disabled - during the export can be re-enabled. */ - wMsg('db-export',{ - filename: fn, - error: e.message - }); - } - return; - } - case 'open': { - /* Expects: { - buffer: ArrayBuffer | Uint8Array, - filename: for logging/informational purposes only - } */ - const opt = ev.data; - let buffer = opt.buffer; - if(buffer instanceof Uint8Array){ - }else if(buffer instanceof ArrayBuffer){ - buffer = new Uint8Array(buffer); - }else{ - stderr("'open' expects {buffer:Uint8Array} containing an uploaded db."); - return; - } - const fn = ( - opt.filename - ? opt.filename.split(/[/\\]/).pop().replace('"','_') - : ("db-"+((Math.random() * 10000000) | 0)+ - "-"+((Math.random() * 10000000) | 0)+".sqlite3") - ); - /* We cannot delete the existing db file until the new one - is installed, which means that we risk overflowing our - quota (if any) by having both the previous and current - db briefly installed in the virtual filesystem. */ - fiddleModule.FS.createDataFile("/", fn, buffer, true, true); - const oldName = Sqlite3Shell.dbFilename(); - Sqlite3Shell.exec('.open "/'+fn+'"'); - if(oldName && oldName !== fn){ - try{fiddleModule.FS.unlink(oldName);} - catch(e){/*ignored*/} + } finally { + delete f._running; + wMsg('working','end'); + } + }, + resetDb: function f(){ + if(!f._) f._ = fiddleModule.cwrap('fiddle_reset_db', null); + stdout("Resetting database."); + f._(); + stdout("Reset",this.dbFilename()); + }, + /* Interrupt can't work: this Worker is tied up working, so won't get the + interrupt event which would be needed to perform the interrupt. */ + interrupt: function f(){ + if(!f._) f._ = fiddleModule.cwrap('fiddle_interrupt', null); + stdout("Requesting interrupt."); + f._(); + } + }; + + self.onmessage = function f(ev){ + ev = ev.data; + if(!f.cache){ + f.cache = { + prevFilename: null + }; + } + //console.debug("worker: onmessage.data",ev); + switch(ev.type){ + case 'shellExec': Sqlite3Shell.exec(ev.data); return; + case 'db-reset': Sqlite3Shell.resetDb(); return; + case 'interrupt': Sqlite3Shell.interrupt(); return; + /** Triggers the export of the current db. Fires an + event in the form: + + {type:'db-export', + data:{ + filename: name of db, + buffer: contents of the db file (Uint8Array), + error: on error, a message string and no buffer property. } - stdout("Replaced DB with",fn+"."); - return; - } - }; - console.warn("Unknown fiddle-worker message type:",ev); - }; - - /** - emscripten module for use with build mode -sMODULARIZE. - */ - const fiddleModule = { - print: stdout, - printErr: stderr, - /** - Intercepts status updates from the emscripting module init - and fires worker events with a type of 'status' and a - payload of: - - { - text: string | null, // null at end of load process - step: integer // starts at 1, increments 1 per call - } - - We have no way of knowing in advance how many steps will - be processed/posted, so creating a "percentage done" view is - not really practical. One can be approximated by giving it a - current value of message.step and max value of message.step+1, - though. - - When work is finished, a message with a text value of null is - submitted. - - After a message with text==null is posted, the module may later - post messages about fatal problems, e.g. an exit() being - triggered, so it is recommended that UI elements for posting - status messages not be outright removed from the DOM when - text==null, and that they instead be hidden until/unless - text!=null. - */ - setStatus: function f(text){ - if(!f.last) f.last = { step: 0, text: '' }; - else if(text === f.last.text) return; - f.last.text = text; - wMsg('module',{ - type:'status', - data:{step: ++f.last.step, text: text||null} + } + */ + case 'db-export': { + const fn = Sqlite3Shell.dbFilename(); + stdout("Exporting",fn+"."); + const fn2 = fn ? fn.split(/[/\\]/).pop() : null; + try{ + if(!fn2) throw new Error("DB appears to be closed."); + wMsg('db-export',{ + filename: fn2, + buffer: fiddleModule.FS.readFile(fn, {encoding:"binary"}) + }); + }catch(e){ + /* Post a failure message so that UI elements disabled + during the export can be re-enabled. */ + wMsg('db-export',{ + filename: fn, + error: e.message }); + } + return; + } + case 'open': { + /* Expects: { + buffer: ArrayBuffer | Uint8Array, + filename: for logging/informational purposes only + } */ + const opt = ev.data; + let buffer = opt.buffer; + if(buffer instanceof Uint8Array){ + }else if(buffer instanceof ArrayBuffer){ + buffer = new Uint8Array(buffer); + }else{ + stderr("'open' expects {buffer:Uint8Array} containing an uploaded db."); + return; + } + const fn = ( + opt.filename + ? opt.filename.split(/[/\\]/).pop().replace('"','_') + : ("db-"+((Math.random() * 10000000) | 0)+ + "-"+((Math.random() * 10000000) | 0)+".sqlite3") + ); + /* We cannot delete the existing db file until the new one + is installed, which means that we risk overflowing our + quota (if any) by having both the previous and current + db briefly installed in the virtual filesystem. */ + fiddleModule.FS.createDataFile("/", fn, buffer, true, true); + const oldName = Sqlite3Shell.dbFilename(); + Sqlite3Shell.exec('.open "/'+fn+'"'); + if(oldName && oldName !== fn){ + try{fiddleModule.fsUnlink(oldName);} + catch(e){/*ignored*/} + } + stdout("Replaced DB with",fn+"."); + return; } }; - - importScripts('fiddle-module.js'); + console.warn("Unknown fiddle-worker message type:",ev); + }; + + /** + emscripten module for use with build mode -sMODULARIZE. + */ + const fiddleModule = { + print: stdout, + printErr: stderr, /** - initFiddleModule() is installed via fiddle-module.js due to - building with: - - emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule + Intercepts status updates from the emscripting module init + and fires worker events with a type of 'status' and a + payload of: + + { + text: string | null, // null at end of load process + step: integer // starts at 1, increments 1 per call + } + + We have no way of knowing in advance how many steps will + be processed/posted, so creating a "percentage done" view is + not really practical. One can be approximated by giving it a + current value of message.step and max value of message.step+1, + though. + + When work is finished, a message with a text value of null is + submitted. + + After a message with text==null is posted, the module may later + post messages about fatal problems, e.g. an exit() being + triggered, so it is recommended that UI elements for posting + status messages not be outright removed from the DOM when + text==null, and that they instead be hidden until/unless + text!=null. */ - initFiddleModule(fiddleModule).then(function(thisModule){ - wMsg('fiddle-ready'); - }); + setStatus: function f(text){ + if(!f.last) f.last = { step: 0, text: '' }; + else if(text === f.last.text) return; + f.last.text = text; + wMsg('module',{ + type:'status', + data:{step: ++f.last.step, text: text||null} + }); + } + }; + + importScripts('fiddle-module.js'); + /** + initFiddleModule() is installed via fiddle-module.js due to + building with: + + emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule + */ + initFiddleModule(fiddleModule).then(function(thisModule){ + thisModule.fsUnlink = thisModule.cwrap('sqlite3_wasm_vfs_unlink','number',['string']); + wMsg('fiddle-ready'); + }); })(); diff --git a/ext/wasm/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js index 619ce4eca..ec56dd593 100644 --- a/ext/wasm/fiddle/fiddle.js +++ b/ext/wasm/fiddle/fiddle.js @@ -730,7 +730,8 @@ -- only that part is executed. -- ================================================ .help`}, - {name: "Timer on", sql: ".timer on"}, + //{name: "Timer on", sql: ".timer on"}, + // ^^^ re-enable if emscripten re-enables getrusage() {name: "Setup table T", sql:`.nullvalue NULL CREATE TABLE t(a,b); INSERT INTO t(a,b) VALUES('abc',123),('def',456),(NULL,789),('ghi',012); @@ -775,7 +776,7 @@ SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;`} }); })()/* example queries */; - SF.echo(null/*clear any output generated by the init process*/); + //SF.echo(null/*clear any output generated by the init process*/); if(window.jQuery && window.jQuery.terminal){ /* Set up the terminal-style view... */ const eTerm = window.jQuery('#view-terminal').empty(); diff --git a/ext/wasm/index.html b/ext/wasm/index.html new file mode 100644 index 000000000..435040434 --- /dev/null +++ b/ext/wasm/index.html @@ -0,0 +1,56 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/testing.css"/> + <title>sqlite3 WASM Testing Page Index</title> + </head> + <body> + <header id='titlebar'><span>sqlite3 WASM test pages</span></header> + <hr> + <div>Below is the list of test pages for the sqlite3 WASM + builds. All of them require that this directory have been + "make"d first. The intent is that <em>this</em> page be run + using:</div> + <blockquote><pre>althttpd -page index.html</pre></blockquote> + <div>and the individual tests be started in their own tab.</div> + <div>Warnings and Caveats: + <ul class='warning'> + <li>Some of these pages require that + the web server emit the so-called COOP and COEP headers. The + default build of althttpd <em>does not</em>. + </li> + <li>Whether or not WASMFS/OPFS support is enabled on any given + page may depend on build-time options which are <em>off by + default</em> because they currently (as of 2022-09-08) break + with Worker-based pages. + </li> + </ul> + </div> + <div>The tests... + <ul id='test-list'> + <li><a href='testing1.html'>testing1</a>: sanity tests of the core APIs and surrounding utility code.</li> + <li><a href='testing2.html'>testing2</a>: Worker-based test of OO API #1.</li> + <li><a href='testing-worker1-promiser.html'>testing-worker1-promiser</a>: + tests for the Promise-based wrapper of the Worker-based API.</li> + <li><a href='batch-runner.html'>batch-runner</a>: runs batches of SQL exported from speedtest1.</li> + <li><a href='speedtest1.html'>speedtest1</a>: a main-thread WASM build of speedtest1.</li> + <li><a href='speedtest1-worker.html'>speedtest1-worker</a>: an interactive Worker-thread variant of speedtest1.</li> + <li><a href='demo-oo1.html'>demo-oo1</a>: demonstration of the OO API #1.</li> + <li><a href='kvvfs1.html'>kvvfs1</a>: very basic demo of using the key-value vfs for storing + a persistent db in JS localStorage or sessionStorage.</li> + <!--li><a href='x.html'></a></li--> + </ul> + </div> + <style> + #test-list { font-size: 120%; } + </style> + <script>//Assign a distinct target tab name for each test page... + document.querySelectorAll('a').forEach(function(e){ + e.target = e.href; + }); + </script> + </body> +</html> diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index a01865857..14c93b3a2 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -394,7 +394,17 @@ self.Jaccwabyt = function StructBinderFactory(config){ const __utf8Decoder = new TextDecoder('utf-8'); const __utf8Encoder = new TextEncoder(); - + /** Internal helper to use in operations which need to distinguish + between SharedArrayBuffer heap memory and non-shared heap. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + const __utf8Decode = function(arrayBuffer, begin, end){ + return __utf8Decoder.decode( + (arrayBuffer.buffer instanceof __SAB) + ? arrayBuffer.slice(begin, end) + : arrayBuffer.subarray(begin, end) + ); + }; /** Uses __lookupMember() to find the given obj.structInfo key. Returns that member if it is a string, else returns false. If the @@ -437,8 +447,7 @@ self.Jaccwabyt = function StructBinderFactory(config){ //log("mem[",pos,"]",mem[pos]); }; //log("addr =",addr,"pos =",pos); - if(addr===pos) return ""; - return __utf8Decoder.decode(new Uint8Array(mem.buffer, addr, pos-addr)); + return (addr===pos) ? "" : __utf8Decode(mem, addr, pos); }; /** diff --git a/ext/wasm/kvvfs.make b/ext/wasm/kvvfs.make index 83a269137..a65f2faf2 100644 --- a/ext/wasm/kvvfs.make +++ b/ext/wasm/kvvfs.make @@ -30,7 +30,7 @@ kvvfs.flags = ######################################################################## # emcc flags for .c/.o. kvvfs.cflags := -kvvfs.cflags += -std=c99 -fPIC +kvvfs.cflags += -std=c99 -fPIC -g kvvfs.cflags += -I. -I$(dir.top) kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT) @@ -38,6 +38,7 @@ kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT) # emcc flags specific to building the final .js/.wasm file... kvvfs.jsflags := -fPIC kvvfs.jsflags += --no-entry +kvvfs.jsflags += --minify 0 kvvfs.jsflags += -sENVIRONMENT=web kvvfs.jsflags += -sMODULARIZE kvvfs.jsflags += -sSTRICT_JS @@ -80,3 +81,4 @@ endif @ls -la $@ $(kvvfs.wasm) kvvfs: $(kvvfs.js) +all: kvvfs diff --git a/ext/wasm/kvvfs1.html b/ext/wasm/kvvfs1.html index 0657920c5..773de0a60 100644 --- a/ext/wasm/kvvfs1.html +++ b/ext/wasm/kvvfs1.html @@ -24,7 +24,16 @@ <div class="emscripten"> <progress value="0" max="100" id="module-progress" hidden='1'></progress> </div><!-- /emscripten bits --> - <div>Everything on this page happens in the dev console.</div> + <div>Everything on this page happens in the dev console. TODOs for this demo include, + but are not necessarily limited to: + + <ul> + <li>UI controls to switch between localStorage and sessionStorage</li> + <li>Button to clear storage.</li> + <li>Button to dump the current db contents.</li> + <!--li></li--> + </ul> + </div> <hr> <div id='test-output'></div> <script src="sqlite3-kvvfs.js"></script> diff --git a/ext/wasm/kvvfs1.js b/ext/wasm/kvvfs1.js index f56f4874e..169fcc8bd 100644 --- a/ext/wasm/kvvfs1.js +++ b/ext/wasm/kvvfs1.js @@ -35,29 +35,21 @@ wasm = capi.wasm; log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); log("Build options:",wasm.compileOptionUsed()); - self.S = sqlite3; - T.assert(0 === capi.sqlite3_vfs_find(null)); - S.capi.sqlite3_initialize(); - T.assert( Number.isFinite( capi.sqlite3_vfs_find(null) ) ); - const stores = { - local: localStorage, - session: sessionStorage - }; - const cleanupStore = function(n){ - const s = stores[n]; - const isKv = (key)=>key.startsWith('kvvfs-'+n); - let i, k, toRemove = []; - for( i = 0; (k = s.key(i)); ++i) { - if(isKv(k)) toRemove.push(k); - } - toRemove.forEach((k)=>s.removeItem(k)); - }; - const dbStorage = 1 ? 'session' : 'local'; - const db = new oo.DB(dbStorage); + T.assert( 0 !== capi.sqlite3_vfs_find(null) ); + + const dbStorage = 1 ? ':sessionStorage:' : ':localStorage:'; + /** + The names ':sessionStorage:' and ':localStorage:' are handled + via the DB class constructor, not the C level. In the C API, + the names "local" and "session" are the current (2022-09-12) + names for those keys, but that is subject to change. + */ + const db = new oo.DB( dbStorage ); + log("Storage backend:",db.filename /* note that the name was internally translated */); try { db.exec("create table if not exists t(a)"); if(undefined===db.selectValue("select a from t limit 1")){ - log("New db. Populating.."); + log("New db. Populating. This DB will persist across page reloads."); db.exec("insert into t(a) values(1),(2),(3)"); }else{ log("Found existing table data:"); @@ -68,12 +60,9 @@ }); } }finally{ - const n = db.filename; db.close(); - //cleanupStore(n); } - - log("Init done. Proceed from the dev console."); + log("End of demo."); }; sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ diff --git a/ext/wasm/scratchpad-opfs-main.html b/ext/wasm/scratchpad-opfs-main.html new file mode 100644 index 000000000..8edd15a67 --- /dev/null +++ b/ext/wasm/scratchpad-opfs-main.html @@ -0,0 +1,40 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>sqlite3 WASMFS/OPFS Main-thread Scratchpad</title> + </head> + <body> + <header id='titlebar'><span>sqlite3 WASMFS/OPFS Main-thread Scratchpad</span></header> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <p>Scratchpad/test app for the WASMF/OPFS integration in the + main window thread. This page requires that the sqlite3 API have + been built with WASMFS support. If OPFS support is available then + it "should" persist a database across reloads (watch the dev console + output), otherwise it will not. + </p> + <p>All stuff on this page happens in the dev console.</p> + <hr> + <div id='test-output'></div> + <script src="sqlite3.js"></script> + <script src="common/SqliteTestUtil.js"></script> + <script src="scratchpad-opfs-main.js"></script> + </body> +</html> diff --git a/ext/wasm/scratchpad-opfs-main.js b/ext/wasm/scratchpad-opfs-main.js new file mode 100644 index 000000000..9f7d3fb1d --- /dev/null +++ b/ext/wasm/scratchpad-opfs-main.js @@ -0,0 +1,73 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const log = console.log.bind(console), + warn = console.warn.bind(console), + error = console.error.bind(console); + + const stdout = log; + const stderr = error; + + const test1 = function(db){ + db.exec("create table if not exists t(a);") + .transaction(function(db){ + db.prepare("insert into t(a) values(?)") + .bind(new Date().getTime()) + .stepFinalize(); + stdout("Number of values in table t:", + db.selectValue("select count(*) from t")); + }); + }; + + const runTests = function(Module){ + //stdout("Module",Module); + self._MODULE = Module /* this is only to facilitate testing from the console */; + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + const persistentDir = capi.sqlite3_web_persistent_dir(); + if(persistentDir){ + stdout("Persistent storage dir:",persistentDir); + }else{ + stderr("No persistent storage available."); + } + const startTime = performance.now(); + let db; + try { + db = new oo.DB(persistentDir+'/foo.db'); + stdout("DB filename:",db.filename,db.fileName()); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + test1 + ].forEach((f)=>{ + const n = performance.now(); + stdout(banner1,"Running",f.name+"()..."); + f(db, sqlite3, Module); + stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); + }); + }finally{ + if(db) db.close(); + } + stdout("Total test time:",(performance.now() - startTime),"ms"); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(runTests); +})(); diff --git a/ext/wasm/scratchpad-opfs-worker.html b/ext/wasm/scratchpad-opfs-worker.html new file mode 100644 index 000000000..2f3e5b1de --- /dev/null +++ b/ext/wasm/scratchpad-opfs-worker.html @@ -0,0 +1,39 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>sqlite3 WASMFS/OPFS Worker-thread Scratchpad</title> + </head> + <body> + <header id='titlebar'><span>sqlite3 WASMFS/OPFS Worker-thread Scratchpad</span></header> + <!-- emscripten bits --> + <!--figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div--><!-- /emscripten bits --> + <p><strong>This test is known, as of 2022-08-13, to not work.</strong></p> + <p>Scratchpad/test app for the WASMF/OPFS integration in a + WORKER thread. This page requires that the sqlite3 API have + been built with WASMFS support. If OPFS support is available then + it "should" persist a database across reloads (watch the dev console + output), otherwise it will not. + </p> + <p>All stuff on this page happens in the dev console.</p> + <hr> + <div id='test-output'></div> + <script src="scratchpad-opfs-worker.js"></script> + </body> +</html> diff --git a/ext/wasm/scratchpad-opfs-worker.js b/ext/wasm/scratchpad-opfs-worker.js new file mode 100644 index 000000000..debd0245e --- /dev/null +++ b/ext/wasm/scratchpad-opfs-worker.js @@ -0,0 +1,32 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread. It will load sqlite3.js in a worker thread. +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + const log = console.log.bind(console), + warn = console.warn.bind(console), + error = console.error.bind(console); + const W = new Worker("scratchpad-opfs-worker2.js"); + self.onmessage = function(ev){ + ev = ev.data; + const d = ev.data; + switch(ev.type){ + case 'stdout': log(d); break; + case 'stderr': error(d); break; + default: warn("Unhandled message type:",ev); break; + } + }; +})(); diff --git a/ext/wasm/scratchpad-opfs-worker2.js b/ext/wasm/scratchpad-opfs-worker2.js new file mode 100644 index 000000000..e27fe2b37 --- /dev/null +++ b/ext/wasm/scratchpad-opfs-worker2.js @@ -0,0 +1,86 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + An experiment for wasmfs/opfs. This file MUST be in the same dir as + the sqlite3.js emscripten module or that module won't be able to + resolve the relative URIs (importScript()'s relative URI handling + is, quite frankly, broken). +*/ +'use strict'; +(function(){ + const toss = function(...args){throw new Error(args.join(' '))}; + importScripts('sqlite3.js'); + + /** + Posts a message in the form {type,data} unless passed more than 2 + args, in which case it posts {type, data:[arg1...argN]}. + */ + const wMsg = function(type,data){ + postMessage({ + type, + data: arguments.length<3 + ? data + : Array.prototype.slice.call(arguments,1) + }); + }; + + const stdout = console.log.bind(console); + const stderr = console.error.bind(console);//function(...args){wMsg('stderr', args);}; + + const test1 = function(db){ + db.execMulti("create table if not exists t(a);") + .transaction(function(db){ + db.prepare("insert into t(a) values(?)") + .bind(new Date().getTime()) + .stepFinalize(); + stdout("Number of values in table t:", + db.selectValue("select count(*) from t")); + }); + }; + + const runTests = function(Module){ + //stdout("Module",Module); + self._MODULE = Module /* this is only to facilitate testing from the console */; + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + const persistentDir = capi.sqlite3_web_persistent_dir(); + if(persistentDir){ + stderr("Persistent storage dir:",persistentDir); + }else{ + stderr("No persistent storage available."); + } + const startTime = performance.now(); + let db; + try { + db = new oo.DB(persistentDir+'/foo.db'); + stdout("DB filename:",db.filename,db.fileName()); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + test1 + ].forEach((f)=>{ + const n = performance.now(); + stdout(banner1,"Running",f.name+"()..."); + f(db, sqlite3, Module); + stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); + }); + }finally{ + if(db) db.close(); + } + stdout("Total test time:",(performance.now() - startTime),"ms"); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(runTests); +})(); diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html new file mode 100644 index 000000000..ba74d9f7c --- /dev/null +++ b/ext/wasm/speedtest1-worker.html @@ -0,0 +1,340 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>speedtest1.wasm Worker</title> + </head> + <body> + <header id='titlebar'>speedtest1.wasm Worker</header> + <div>See also: <a href='speedtest1.html'>A main-thread variant of this page.</a></div> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <fieldset id='ui-controls' class='hidden'> + <legend>Options</legend> + <div id='toolbar'> + <div id='toolbar-select'> + <select id='select-flags' size='10' multiple></select> + <div>TODO? Options which require values are not represented here.</div> + </div> + <div class='toolbar-inner-vertical'> + <div id='toolbar-selected-flags'></div> + <span>→ <a id='link-main-thread' href='#' target='main-thread' + title='Start speedtest1.html with the selected flags'>speedtest1.html</a> + </span> + </div> + <div class='toolbar-inner-vertical' id='toolbar-runner-controls'> + <button id='btn-reset-flags'>Reset Flags</button> + <button id='btn-output-clear'>Clear output</button> + <button id='btn-run'>Run</button> + </div> + </div> + </fieldset> + <div> + <span class='input-wrapper'> + <input type='checkbox' class='disable-during-eval' id='cb-reverse-log-order' checked></input> + <label for='cb-reverse-log-order' id='lbl-reverse-log-order'>Reverse log order</label> + </span> + </div> + <div id='test-output'> + </div> + <div id='tips'> + <strong>Tips:</strong> + <ul> + <li>Control-click the flags to (de)select multiple flags.</li> + <li>The <tt>--big-transactions</tt> flag is important for two + of the bigger tests. Without it, those tests create a + combined total of 140k implicit transactions, reducing their + speed to an absolute crawl, especially when WASMFS is + activated. + </li> + <li>The easiest way to try different optimization levels is, + from this directory: + <pre>$ rm -f speedtest1.js; make -e emcc_opt='-O2' speedtest1.js</pre> + Then reload this page. -O2 seems to consistently produce the fastest results. + </li> + </ul> + </div> + <style> + #test-output { + white-space: break-spaces; + overflow: auto; + } + div#tips { margin-top: 1em; } + #toolbar { + display: flex; + flex-direction: row; + flex-wrap: wrap; + } + #toolbar > * { + margin: 0 0.5em; + } + .toolbar-inner-vertical { + display: flex; + flex-direction: column; + justify-content: space-between; + } + #toolbar-select { + display: flex; + flex-direction: column; + } + .toolbar-inner-vertical > *, #toolbar-select > * { + margin: 0.2em 0; + } + #select-flags > option { + white-space: pre; + font-family: monospace; + } + fieldset { + border-radius: 0.5em; + } + #toolbar-runner-controls { flex-grow: 1 } + #toolbar-runner-controls > * { flex: 1 0 auto } + #toolbar-selected-flags::before { + font-family: initial; + content:"Selected flags: "; + } + #toolbar-selected-flags { + display: flex; + flex-direction: column; + font-family: monospace; + justify-content: flex-start; + } + </style> + <script>(function(){ + 'use strict'; + const E = (sel)=>document.querySelector(sel); + const eOut = E('#test-output'); + const log2 = function(cssClass,...args){ + let ln; + if(1 || cssClass){ + ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + }else{ + // This doesn't work with the "reverse order" option! + ln = document.createTextNode(args.join(' ')+'\n'); + } + eOut.append(ln); + }; + const log = (...args)=>{ + //console.log(...args); + log2('', ...args); + }; + const logErr = function(...args){ + //console.error(...args); + log2('error', ...args); + }; + const logWarn = function(...args){ + //console.warn(...args); + log2('warning', ...args); + }; + + const spacePad = function(str,len=21){ + if(str.length===len) return str; + else if(str.length>len) return str.substr(0,len); + const a = []; a.length = len - str.length; + return str+a.join(' '); + }; + // OPTION elements seem to ignore white-space:pre, so do this the hard way... + const nbspPad = function(str,len=21){ + if(str.length===len) return str; + else if(str.length>len) return str.substr(0,len); + const a = []; a.length = len - str.length; + return str+a.join(' '); + }; + + const W = new Worker("speedtest1-worker.js"); + const mPost = function(msgType,payload){ + W.postMessage({type: msgType, data: payload}); + }; + + const eFlags = E('#select-flags'); + const eSelectedFlags = E('#toolbar-selected-flags'); + const eLinkMainThread = E('#link-main-thread'); + const getSelectedFlags = ()=>Array.prototype.map.call(eFlags.selectedOptions, (v)=>v.value); + const updateSelectedFlags = function(){ + eSelectedFlags.innerText = ''; + const flags = getSelectedFlags(); + flags.forEach(function(f){ + const e = document.createElement('span'); + e.innerText = f; + eSelectedFlags.appendChild(e); + }); + const rxStripDash = /^(-+)?/; + const comma = flags.map((v)=>v.replace(rxStripDash,'')).join(','); + eLinkMainThread.setAttribute('target', 'main-thread-'+comma); + eLinkMainThread.href = 'speedtest1.html?flags='+comma; + }; + eFlags.addEventListener('change', updateSelectedFlags ); + { + const flags = Object.create(null); + /* TODO? Flags which require values need custom UI + controls and some of them make little sense here + (e.g. --script FILE). */ + flags["autovacuum"] = "Enable AUTOVACUUM mode"; + flags["big-transactions"] = "Important for tests 410 and 510!"; + //flags["cachesize"] = "N Set the cache size to N"; + flags["checkpoint"] = "Run PRAGMA wal_checkpoint after each test case"; + flags["exclusive"] = "Enable locking_mode=EXCLUSIVE"; + flags["explain"] = "Like --sqlonly but with added EXPLAIN keywords"; + //flags["heap"] = "SZ MIN Memory allocator uses SZ bytes & min allocation MIN"; + flags["incrvacuum"] = "Enable incremenatal vacuum mode"; + //flags["journal"] = "M Set the journal_mode to M"; + //flags["key"] = "KEY Set the encryption key to KEY"; + //flags["lookaside"] = "N SZ Configure lookaside for N slots of SZ bytes each"; + flags["memdb"] = "Use an in-memory database"; + //flags["mmap"] = "SZ MMAP the first SZ bytes of the database file"; + flags["multithread"] = "Set multithreaded mode"; + flags["nomemstat"] = "Disable memory statistics"; + flags["nomutex"] = "Open db with SQLITE_OPEN_NOMUTEX"; + flags["nosync"] = "Set PRAGMA synchronous=OFF"; + flags["notnull"] = "Add NOT NULL constraints to table columns"; + //flags["output"] = "FILE Store SQL output in FILE"; + //flags["pagesize"] = "N Set the page size to N"; + //flags["pcache"] = "N SZ Configure N pages of pagecache each of size SZ bytes"; + //flags["primarykey"] = "Use PRIMARY KEY instead of UNIQUE where appropriate"; + //flags["repeat"] = "N Repeat each SELECT N times (default: 1)"; + flags["reprepare"] = "Reprepare each statement upon every invocation"; + //flags["reserve"] = "N Reserve N bytes on each database page"; + //flags["script"] = "FILE Write an SQL script for the test into FILE"; + flags["serialized"] = "Set serialized threading mode"; + flags["singlethread"] = "Set single-threaded mode - disables all mutexing"; + flags["sqlonly"] = "No-op. Only show the SQL that would have been run."; + flags["shrink"] = "memory Invoke sqlite3_db_release_memory() frequently."; + //flags["size"] = "N Relative test size. Default=100"; + flags["strict"] = "Use STRICT table where appropriate"; + flags["stats"] = "Show statistics at the end"; + //flags["temp"] = "N N from 0 to 9. 0: no temp table. 9: all temp tables"; + //flags["testset"] = "T Run test-set T (main, cte, rtree, orm, fp, debug)"; + flags["trace"] = "Turn on SQL tracing"; + //flags["threads"] = "N Use up to N threads for sorting"; + /* + The core API's WASM build does not support UTF16, but in + this app it's not an issue because the data are not crossing + JS/WASM boundaries. + */ + flags["utf16be"] = "Set text encoding to UTF-16BE"; + flags["utf16le"] = "Set text encoding to UTF-16LE"; + flags["verify"] = "Run additional verification steps."; + flags["without"] = "rowid Use WITHOUT ROWID where appropriate"; + const preselectedFlags = [ + 'big-transactions', + 'memdb', + 'singlethread' + ]; + Object.keys(flags).sort().forEach(function(f){ + const opt = document.createElement('option'); + eFlags.appendChild(opt); + const lbl = nbspPad('--'+f)+flags[f]; + //opt.innerText = lbl; + opt.innerHTML = lbl; + opt.value = '--'+f; + if(preselectedFlags.indexOf(f) >= 0) opt.selected = true; + }); + + const cbReverseLog = E('#cb-reverse-log-order'); + const lblReverseLog = E('#lbl-reverse-log-order'); + if(cbReverseLog.checked){ + lblReverseLog.classList.add('warning'); + eOut.classList.add('reverse'); + } + cbReverseLog.addEventListener('change', function(){ + if(this.checked){ + eOut.classList.add('reverse'); + lblReverseLog.classList.add('warning'); + }else{ + eOut.classList.remove('reverse'); + lblReverseLog.classList.remove('warning'); + } + }, false); + updateSelectedFlags(); + } + E('#btn-output-clear').addEventListener('click', ()=>{ + eOut.innerText = ''; + }); + E('#btn-reset-flags').addEventListener('click',()=>{ + eFlags.value = ''; + updateSelectedFlags(); + }); + E('#btn-run').addEventListener('click',function(){ + log("Running speedtest1. UI controls will be disabled until it completes."); + mPost('run', getSelectedFlags()); + }); + + const eControls = E('#ui-controls'); + /** Update Emscripten-related UI elements while loading the module. */ + const updateLoadStatus = function f(text){ + if(!f.last){ + f.last = { text: '', step: 0 }; + const E = (cssSelector)=>document.querySelector(cssSelector); + f.ui = { + status: E('#module-status'), + progress: E('#module-progress'), + spinner: E('#module-spinner') + }; + } + if(text === f.last.text) return; + f.last.text = text; + if(f.ui.progress){ + f.ui.progress.value = f.last.step; + f.ui.progress.max = f.last.step + 1; + } + ++f.last.step; + if(text) { + f.ui.status.classList.remove('hidden'); + f.ui.status.innerText = text; + }else{ + if(f.ui.progress){ + f.ui.progress.remove(); + f.ui.spinner.remove(); + delete f.ui.progress; + delete f.ui.spinner; + } + f.ui.status.classList.add('hidden'); + } + }; + + W.onmessage = function(msg){ + msg = msg.data; + switch(msg.type){ + case 'ready': + log("Worker is ready."); + eControls.classList.remove('hidden'); + break; + case 'stdout': log(msg.data); break; + case 'stdout': logErr(msg.data); break; + case 'run-start': + eControls.disabled = true; + log("Running speedtest1 with argv =",msg.data.join(' ')); + break; + case 'run-end': + log("speedtest1 finished."); + eControls.disabled = false; + // app output is in msg.data + break; + case 'error': logErr(msg.data); break; + case 'load-status': updateLoadStatus(msg.data); break; + default: + logErr("Unhandled worker message type:",msg); + break; + } + }; + })();</script> + </body> +</html> diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js new file mode 100644 index 000000000..8512bdbbf --- /dev/null +++ b/ext/wasm/speedtest1-worker.js @@ -0,0 +1,99 @@ +'use strict'; +(function(){ + importScripts('common/whwasmutil.js','speedtest1.js'); + /** + If this environment contains OPFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it returns + an empty string. + */ + const opfsDir = function f(wasmUtil){ + if(undefined !== f._) return f._; + const pdir = '/persistent'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_opfs() is not available + return f._ = ""; + } + }; + opfsDir._ = undefined; + + const mPost = function(msgType,payload){ + postMessage({type: msgType, data: payload}); + }; + + const App = Object.create(null); + App.logBuffer = []; + const logMsg = (type,msgArgs)=>{ + const msg = msgArgs.join(' '); + App.logBuffer.push(msg); + mPost(type,msg); + }; + const log = (...args)=>logMsg('stdout',args); + const logErr = (...args)=>logMsg('stderr',args); + + const runSpeedtest = function(cliFlagsArray){ + const scope = App.wasm.scopedAllocPush(); + const dbFile = 0 ? "" : App.pDir+"/speedtest1.db"; + try{ + const argv = [ + "speedtest1.wasm", ...cliFlagsArray, dbFile + ]; + App.logBuffer.length = 0; + mPost('run-start', [...argv]); + App.wasm.xCall('__main_argc_argv', argv.length, + App.wasm.scopedAllocMainArgv(argv)); + }catch(e){ + mPost('error',e.message); + }finally{ + App.wasm.scopedAllocPop(scope); + App.unlink(dbFile); + mPost('run-end', App.logBuffer.join('\n')); + App.logBuffer.length = 0; + } + }; + + self.onmessage = function(msg){ + msg = msg.data; + switch(msg.type){ + case 'run': runSpeedtest(msg.data || []); break; + default: + logErr("Unhandled worker message type:",msg.type); + break; + } + }; + + const EmscriptenModule = { + print: log, + printErr: logErr, + setStatus: (text)=>mPost('load-status',text) + }; + self.sqlite3Speedtest1InitModule(EmscriptenModule).then(function(EmscriptenModule){ + log("Module inited."); + App.wasm = { + exports: EmscriptenModule.asm, + alloc: (n)=>EmscriptenModule._malloc(n), + dealloc: (m)=>EmscriptenModule._free(m), + memory: EmscriptenModule.asm.memory || EmscriptenModule.wasmMemory + }; + //console.debug('wasm =',wasm); + self.WhWasmUtilInstaller(App.wasm); + App.unlink = App.wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]); + App.pDir = opfsDir(App.wasm); + if(App.pDir){ + log("Persistent storage:",pDir); + } + mPost('ready',true); + }); +})(); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html new file mode 100644 index 000000000..bc8b4ec9f --- /dev/null +++ b/ext/wasm/speedtest1.html @@ -0,0 +1,154 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>speedtest1.wasm</title> + </head> + <body> + <header id='titlebar'><span>speedtest1.wasm</span></header> + <div>See also: <a href='speedtest1-worker.html'>A Worker-thread variant of this page.</a></div> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <div class='warning'>This page starts running the main exe when it loads, which will + block the UI until it finishes! Adding UI controls to manually configure and start it + are TODO.</div> + </div> + <div class='warning'>Achtung: running it with the dev tools open may + <em>drastically</em> slow it down. For faster results, keep the dev + tools closed when running it! + </div> + <div>Output is delayed/buffered because we cannot update the UI while the + speedtest is running. Output will appear below when ready... + <div id='test-output'></div> + <script src="common/whwasmutil.js"></script> + <script src="common/SqliteTestUtil.js"></script> + <script src="speedtest1.js"></script> + <script>(function(){ + /** + If this environment contains OPFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it returns + an empty string. + */ + const opfsDir = function f(wasmUtil){ + if(undefined !== f._) return f._; + const pdir = '/persistent'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_opfs() is not available + return f._ = ""; + } + }; + opfsDir._ = undefined; + + const eOut = document.querySelector('#test-output'); + const log2 = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOut.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + }; + const logList = []; + const dumpLogList = function(){ + logList.forEach((v)=>log2('',v)); + logList.length = 0; + }; + /* can't update DOM while speedtest is running unless we run + speedtest in a worker thread. */; + const log = (...args)=>{ + console.log(...args); + logList.push(args.join(' ')); + }; + const logErr = function(...args){ + console.error(...args); + logList.push('ERROR: '+args.join(' ')); + }; + + const runTests = function(EmscriptenModule){ + console.log("Module inited.",EmscriptenModule); + const wasm = { + exports: EmscriptenModule.asm, + alloc: (n)=>EmscriptenModule._malloc(n), + dealloc: (m)=>EmscriptenModule._free(m), + memory: EmscriptenModule.asm.memory || EmscriptenModule.wasmMemory + }; + //console.debug('wasm =',wasm); + self.WhWasmUtilInstaller(wasm); + const unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]); + const pDir = opfsDir(wasm); + if(pDir){ + console.warn("Persistent storage:",pDir); + } + const scope = wasm.scopedAllocPush(); + const dbFile = 0 ? "" : pDir+"/speedtest1.db"; + const urlArgs = self.SqliteTestUtil.processUrlArgs(); + const argv = ["speedtest1"]; + if(urlArgs.flags){ + // transform flags=a,b,c to ["--a", "--b", "--c"] + argv.push(...(urlArgs.flags.split(',').map((v)=>'--'+v))); + }else{ + argv.push( + "--singlethread", + "--nomutex", + "--nosync", + "--nomemstat" + ); + //"--memdb", // note that memdb trumps the filename arg + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + } + console.log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. + if(pDir) unlink(dbFile); + log2('',"Starting native app:\n ",argv.join(' ')); + log2('',"This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + logList.length = 0; + setTimeout(function(){ + wasm.xCall('__main_argc_argv', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if(pDir) unlink(dbFile); + logList.unshift("Done running native main(). Output:"); + dumpLogList(); + }, 50); + }/*runTests()*/; + + self.sqlite3TestModule.print = log; + self.sqlite3TestModule.printErr = logErr; + sqlite3Speedtest1InitModule(self.sqlite3TestModule).then(function(M){ + setTimeout(()=>runTests(M), 100); + }); + })(); + </script> + </body> +</html> diff --git a/ext/wasm/split-speedtest1-script.sh b/ext/wasm/split-speedtest1-script.sh new file mode 100755 index 000000000..e072d08a1 --- /dev/null +++ b/ext/wasm/split-speedtest1-script.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Expects $1 to be a (speedtest1 --script) output file. Output is a +# series of SQL files extracted from that file. +infile=${1:?arg = speedtest1 --script output file} +testnums=$(grep -e '^-- begin test' "$infile" | cut -d' ' -f4) +if [ x = "x${testnums}" ]; then + echo "Could not parse any begin/end blocks out of $infile" 1>&2 + exit 1 +fi +odir=${infile%%/*} +if [ "$odir" = "$infile" ]; then odir="."; fi +#echo testnums=$testnums +for n in $testnums; do + ofile=$odir/$(printf "speedtest1-%03d.sql" $n) + sed -n -e "/^-- begin test $n /,/^-- end test $n\$/p" $infile > $ofile + echo -e "$n\t$ofile" +done diff --git a/ext/wasm/sql/000-mandelbrot.sql b/ext/wasm/sql/000-mandelbrot.sql new file mode 100644 index 000000000..3aa5f5715 --- /dev/null +++ b/ext/wasm/sql/000-mandelbrot.sql @@ -0,0 +1,17 @@ +WITH RECURSIVE + xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2), + yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0), + m(iter, cx, cy, x, y) AS ( + SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis + UNION ALL + SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m + WHERE (x*x + y*y) < 4.0 AND iter<28 + ), + m2(iter, cx, cy) AS ( + SELECT max(iter), cx, cy FROM m GROUP BY cx, cy + ), + a(t) AS ( + SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') + FROM m2 GROUP BY cy + ) +SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a; diff --git a/ext/wasm/sql/001-sudoku.sql b/ext/wasm/sql/001-sudoku.sql new file mode 100644 index 000000000..53661b1c3 --- /dev/null +++ b/ext/wasm/sql/001-sudoku.sql @@ -0,0 +1,28 @@ +WITH RECURSIVE + input(sud) AS ( + VALUES('53..7....6..195....98....6.8...6...34..8.3..17...2...6.6....28....419..5....8..79') + ), + digits(z, lp) AS ( + VALUES('1', 1) + UNION ALL SELECT + CAST(lp+1 AS TEXT), lp+1 FROM digits WHERE lp<9 + ), + x(s, ind) AS ( + SELECT sud, instr(sud, '.') FROM input + UNION ALL + SELECT + substr(s, 1, ind-1) || z || substr(s, ind+1), + instr( substr(s, 1, ind-1) || z || substr(s, ind+1), '.' ) + FROM x, digits AS z + WHERE ind>0 + AND NOT EXISTS ( + SELECT 1 + FROM digits AS lp + WHERE z.z = substr(s, ((ind-1)/9)*9 + lp, 1) + OR z.z = substr(s, ((ind-1)%9) + (lp-1)*9 + 1, 1) + OR z.z = substr(s, (((ind-1)/3) % 3) * 3 + + ((ind-1)/27) * 27 + lp + + ((lp-1) / 3) * 6, 1) + ) + ) +SELECT s FROM x WHERE ind=0; diff --git a/ext/wasm/sqlite3-worker1-promiser.js b/ext/wasm/sqlite3-worker1-promiser.js new file mode 100644 index 000000000..7327e14c7 --- /dev/null +++ b/ext/wasm/sqlite3-worker1-promiser.js @@ -0,0 +1,255 @@ +/* + 2022-08-24 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Promise-based proxy for the sqlite3 Worker + API #1. It is intended to be included either from the main thread or + a Worker, but only if (A) the environment supports nested Workers + and (B) it's _not_ a Worker which loads the sqlite3 WASM/JS + module. This file's features will load that module and provide a + slightly simpler client-side interface than the slightly-lower-level + Worker API does. + + This script necessarily exposes one global symbol, but clients may + freely `delete` that symbol after calling it. +*/ +'use strict'; +/** + Configures an sqlite3 Worker API #1 Worker such that it can be + manipulated via a Promise-based interface and returns a factory + function which returns Promises for communicating with the worker. + This proxy has an _almost_ identical interface to the normal + worker API, with any exceptions documented below. + + It requires a configuration object with the following properties: + + - `worker` (required): a Worker instance which loads + `sqlite3-worker1.js` or a functional equivalent. Note that this + function replaces the worker.onmessage property. This property + may alternately be a function, in which case this function + re-assigns this property with the result of calling that + function, enabling delayed instantiation of a Worker. + + - `onready` (optional, but...): this callback is called with no + arguments when the worker fires its initial + 'sqlite3-api'/'worker1-ready' message, which it does when + sqlite3.initWorker1API() completes its initialization. This is + the simplest way to tell the worker to kick of work at the + earliest opportunity. + + - `onerror` (optional): a callback to pass error-type events from + the worker. The object passed to it will be the error message + payload from the worker. This is _not_ the same as the + worker.onerror property! + + - `onunhandled` (optional): a callback which gets passed the + message event object for any worker.onmessage() events which + are not handled by this proxy. Ideally that "should" never + happen, as this proxy aims to handle all known message types. + + - `generateMessageId` (optional): a function which, when passed + an about-to-be-posted message object, generates a _unique_ + message ID for the message, which this API then assigns as the + messageId property of the message. It _must_ generate unique + IDs so that dispatching can work. If not defined, a default + generator is used. + + - `dbId` (optional): is the database ID to be used by the + worker. This must initially be unset or a falsy value. The + first `open` message sent to the worker will cause this config + entry to be assigned to the ID of the opened database. That ID + "should" be set as the `dbId` property of the message sent in + future requests, so that the worker uses that database. + However, if the worker is not given an explicit dbId, it will + use the first-opened database by default. If client code needs + to work with multiple database IDs, the client-level code will + need to juggle those themselves. A `close` message will clear + this property if it matches the ID of the closed db. Potential + TODO: add a config callback specifically for reporting `open` + and `close` message results, so that clients may track those + values. + + - `debug` (optional): a console.debug()-style function for logging + information about messages. + + + This function returns a stateful factory function with the + following interfaces: + + - Promise function(messageType, messageArgs) + - Promise function({message object}) + + The first form expects the "type" and "args" values for a Worker + message. The second expects an object in the form {type:..., + args:...} plus any other properties the client cares to set. This + function will always set the messageId property on the object, + even if it's already set, and will set the dbId property to + config.dbId if it is _not_ set in the message object. + + The function throws on error. + + The function installs a temporarily message listener, posts a + message to the configured Worker, and handles the message's + response via the temporary message listener. The then() callback + of the returned Promise is passed the `message.data` property from + the resulting message, i.e. the payload from the worker, stripped + of the lower-level event state which the onmessage() handler + receives. + + Example usage: + + ``` + const config = {...}; + const eventPromiser = sqlite3Worker1Promiser(config); + eventPromiser('open', {filename:"/foo.db"}).then(function(msg){ + console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...} + // Recall that config.dbId will be set for the first 'open' + // call and cleared for a matching 'close' call. + }); + eventPromiser({type:'close'}).then((msg)=>{ + console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...} + // Recall that config.dbId will be used by default for the message's dbId if + // none is explicitly provided, and a 'close' op will clear config.dbId if it + // closes that exact db. + }); + ``` + + Differences from Worker API #1: + + - exec's {callback: STRING} option does not work via this + interface (it triggers an exception), but {callback: function} + does and works exactly like the STRING form does in the Worker: + the callback is called one time for each row of the result set, + passed the same worker message format as the worker API emits: + + {type:typeString, row:VALUE, rowNumber:1-based-#} + + Where `typeString` is an internally-synthesized message type string + used temporarily for worker message dispatching. It can be ignored + by all client code except that which tests this API. The `row` + property contains the row result in the form implied by the + `rowMode` option (defaulting to `'array'`). The `rowNumber` is a + 1-based integer value incremented by 1 on each call into th + callback. + + At the end of the result set, the same event is fired with + (row=undefined, rowNumber=null) to indicate that + the end of the result set has been reached. Note that the rows + arrive via worker-posted messages, with all the implications + of that. +*/ +self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ + // Inspired by: https://stackoverflow.com/a/52439530 + const handlerMap = Object.create(null); + const noop = function(){}; + const err = config.onerror || noop; + const debug = config.debug || noop; + const idTypeMap = config.generateMessageId ? undefined : Object.create(null); + const genMsgId = config.generateMessageId || function(msg){ + return msg.type+'#'+(idTypeMap[msg.type] = (idTypeMap[msg.type]||0) + 1); + }; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function'===typeof config.worker) config.worker = config.worker(); + config.worker.onmessage = function(ev){ + ev = ev.data; + debug('worker1.onmessage',ev); + let msgHandler = handlerMap[ev.messageId]; + if(!msgHandler){ + if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { + /*fired one time when the Worker1 API initializes*/ + if(config.onready) config.onready(); + return; + } + msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; + if(msgHandler && msgHandler.onrow){ + msgHandler.onrow(ev); + return; + } + if(config.onunhandled) config.onunhandled(arguments[0]); + else err("sqlite3Worker1Promiser() unhandled worker message:",ev); + return; + } + delete handlerMap[ev.messageId]; + switch(ev.type){ + case 'error': + msgHandler.reject(ev); + return; + case 'open': + if(!config.dbId) config.dbId = ev.dbId; + break; + case 'close': + if(config.dbId === ev.dbId) config.dbId = undefined; + break; + default: + break; + } + try {msgHandler.resolve(ev)} + catch(e){msgHandler.reject(e)} + }/*worker.onmessage()*/; + return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + let msg; + if(1===arguments.length){ + msg = arguments[0]; + }else if(2===arguments.length){ + msg = { + type: arguments[0], + args: arguments[1] + }; + }else{ + toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); + } + if(!msg.dbId) msg.dbId = config.dbId; + msg.messageId = genMsgId(msg); + msg.departureTime = performance.now(); + const proxy = Object.create(null); + proxy.message = msg; + let rowCallbackId /* message handler ID for exec on-row callback proxy */; + if('exec'===msg.type && msg.args){ + if('function'===typeof msg.args.callback){ + rowCallbackId = msg.messageId+':row'; + proxy.onrow = msg.args.callback; + msg.args.callback = rowCallbackId; + handlerMap[rowCallbackId] = proxy; + }else if('string' === typeof msg.args.callback){ + toss("exec callback may not be a string when using the Promise interface."); + /** + Design note: the reason for this limitation is that this + API takes over worker.onmessage() and the client has no way + of adding their own message-type handlers to it. Per-row + callbacks are implemented as short-lived message.type + mappings for worker.onmessage(). + + We "could" work around this by providing a new + config.fallbackMessageHandler (or some such) which contains + a map of event type names to callbacks. Seems like overkill + for now, seeing as the client can pass callback functions + to this interface (whereas the string-form "callback" is + needed for the over-the-Worker interface). + */ + } + } + //debug("requestWork", msg); + let p = new Promise(function(resolve, reject){ + proxy.resolve = resolve; + proxy.reject = reject; + handlerMap[msg.messageId] = proxy; + debug("Posting",msg.type,"message to Worker dbId="+(config.dbId||'default')+':',msg); + config.worker.postMessage(msg); + }); + if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]); + return p; + }; +}/*sqlite3Worker1Promiser()*/; +self.sqlite3Worker1Promiser.defaultConfig = { + worker: ()=>new Worker('sqlite3-worker1.js'), + onerror: (...args)=>console.error('worker1 error',...args), + dbId: undefined +}; diff --git a/ext/wasm/api/sqlite3-worker.js b/ext/wasm/sqlite3-worker1.js index 48797de8a..ff024d821 100644 --- a/ext/wasm/api/sqlite3-worker.js +++ b/ext/wasm/sqlite3-worker1.js @@ -14,7 +14,7 @@ sqlite3.js, initializes the module, and postMessage()'s a message after the module is initialized: - {type: 'sqlite3-api', data: 'worker-ready'} + {type: 'sqlite3-api', result: 'worker1-ready'} This seemingly superfluous level of indirection is necessary when loading sqlite3.js via a Worker. Instantiating a worker with new @@ -28,4 +28,4 @@ */ "use strict"; importScripts('sqlite3.js'); -sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorkerAPI()); +sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorker1API()); diff --git a/ext/wasm/testing-worker1-promiser.html b/ext/wasm/testing-worker1-promiser.html new file mode 100644 index 000000000..9af809d9e --- /dev/null +++ b/ext/wasm/testing-worker1-promiser.html @@ -0,0 +1,34 @@ +<!doctype html> +<html lang="en-us"> + <head> + <meta charset="utf-8"> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> + <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> + <link rel="stylesheet" href="common/emscripten.css"/> + <link rel="stylesheet" href="common/testing.css"/> + <title>worker-promise tests</title> + </head> + <body> + <header id='titlebar'><span>worker-promise tests</span></header> + <!-- emscripten bits --> + <figure id="module-spinner"> + <div class="spinner"></div> + <div class='center'><strong>Initializing app...</strong></div> + <div class='center'> + On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. + </div> + </figure> + <div class="emscripten" id="module-status">Downloading...</div> + <div class="emscripten"> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> + </div><!-- /emscripten bits --> + <div>Most stuff on this page happens in the dev console.</div> + <hr> + <div id='test-output'></div> + <script src="common/SqliteTestUtil.js"></script> + <script src="sqlite3-worker1-promiser.js"></script> + <script src="testing-worker1-promiser.js"></script> + </body> +</html> diff --git a/ext/wasm/testing-worker1-promiser.js b/ext/wasm/testing-worker1-promiser.js new file mode 100644 index 000000000..4cc654788 --- /dev/null +++ b/ext/wasm/testing-worker1-promiser.js @@ -0,0 +1,273 @@ +/* + 2022-08-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + Demonstration of the sqlite3 Worker API #1 Promiser: a Promise-based + proxy for for the sqlite3 Worker #1 API. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const eOutput = document.querySelector('#test-output'); + const warn = console.warn.bind(console); + const error = console.error.bind(console); + const log = console.log.bind(console); + const logHtml = async function(cssClass,...args){ + log.apply(this, args); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + + let startTime; + const testCount = async ()=>{ + logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); + }; + + //why is this triggered even when we catch() a Promise? + //window.addEventListener('unhandledrejection', function(event) { + // warn('unhandledrejection',event); + //}); + + const promiserConfig = { + worker: ()=>{ + const w = new Worker("sqlite3-worker1.js"); + w.onerror = (event)=>error("worker.onerror",event); + return w; + }, + debug: 1 ? undefined : (...args)=>console.debug('worker debug',...args), + onunhandled: function(ev){ + error("Unhandled worker message:",ev.data); + }, + onready: function(){ + self.sqlite3TestModule.setStatus(null)/*hide the HTML-side is-loading spinner*/; + runTests(); + }, + onerror: function(ev){ + error("worker1 error:",ev); + } + }; + const workerPromise = self.sqlite3Worker1Promiser(promiserConfig); + delete self.sqlite3Worker1Promiser; + + const wtest = async function(msgType, msgArgs, callback){ + if(2===arguments.length && 'function'===typeof msgArgs){ + callback = msgArgs; + msgArgs = undefined; + } + const p = workerPromise({type: msgType, args:msgArgs}); + return callback ? p.then(callback).finally(testCount) : p; + }; + + const runTests = async function(){ + const dbFilename = '/testing2.sqlite3'; + startTime = performance.now(); + + let sqConfig; + await wtest('config-get', (ev)=>{ + const r = ev.result; + log('sqlite3.config subset:', r); + T.assert('boolean' === typeof r.bigIntEnabled) + .assert('string'===typeof r.persistentDirName) + .assert('boolean' === typeof r.persistenceEnabled); + sqConfig = r; + }); + logHtml('', + "Sending 'open' message and waiting for its response before continuing..."); + + await wtest('open', { + filename: dbFilename, + persistent: sqConfig.persistenceEnabled, + simulateError: 0 /* if true, fail the 'open' */, + }, function(ev){ + const r = ev.result; + log("then open result",r); + T.assert(r.persistent === sqConfig.persistenceEnabled) + .assert(r.persistent + ? (dbFilename!==r.filename) + : (dbFilename==r.filename)) + .assert(ev.dbId === r.dbId) + .assert(ev.messageId) + .assert(promiserConfig.dbId === ev.dbId); + }).then(runTests2); + }; + + const runTests2 = async function(){ + const mustNotReach = ()=>toss("This is not supposed to be reached."); + + await wtest('exec',{ + sql: ["create table t(a,b)", + "insert into t(a,b) values(1,2),(3,4),(5,6)" + ].join(';'), + multi: true, + resultRows: [], columnNames: [] + }, function(ev){ + ev = ev.result; + T.assert(0===ev.resultRows.length) + .assert(0===ev.columnNames.length); + }); + + await wtest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + }, function(ev){ + ev = ev.result; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0][0]) + .assert(6===ev.resultRows[2][1]) + .assert(2===ev.columnNames.length) + .assert('b'===ev.columnNames[1]); + }); + + await wtest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + rowMode: 'object' + }, function(ev){ + ev = ev.result; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0].a) + .assert(6===ev.resultRows[2].b) + }); + + await wtest( + 'exec', + {sql:'intentional_error'}, + mustNotReach + ).catch((e)=>{ + warn("Intentional error:",e); + // Why does the browser report console.error "Uncaught (in + // promise)" when we catch(), and does so _twice_ if we don't + // catch()? According to all docs, that error must be supressed + // if we explicitly catch(). + }); + + await wtest('exec',{ + sql:'select 1 union all select 3', + resultRows: [], + //rowMode: 'array', // array is the default in the Worker interface + }, function(ev){ + ev = ev.result; + T.assert(2 === ev.resultRows.length) + .assert(1 === ev.resultRows[0][0]) + .assert(3 === ev.resultRows[1][0]); + }); + + const resultRowTest1 = function f(ev){ + if(undefined === f.counter) f.counter = 0; + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(2===ev.columnNames.length) + .assert('a'===ev.columnNames[0]) + .assert('B'===ev.columnNames[1]); + }else{ + T.assert(ev.rowNumber > 0); + ++f.counter; + } + log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row.B); + }; + await wtest('exec',{ + sql: 'select a a, b B from t order by a limit 3', + callback: resultRowTest1, + rowMode: 'object' + }, function(ev){ + T.assert(3===resultRowTest1.counter); + resultRowTest1.counter = 0; + }); + + const resultRowTest2 = function f(ev){ + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(1===ev.columnNames.length) + .assert('a'===ev.columnNames[0]) + }else{ + T.assert(ev.rowNumber > 0); + f.counter = ev.rowNumber; + } + log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row); + }; + await wtest('exec',{ + sql: 'select a a from t limit 3', + callback: resultRowTest2, + rowMode: 0 + }, function(ev){ + T.assert(3===resultRowTest2.counter); + }); + + const resultRowTest3 = function f(ev){ + if(null === ev.rowNumber){ + T.assert(3===ev.columnNames.length) + .assert('foo'===ev.columnNames[0]) + .assert('bar'===ev.columnNames[1]) + .assert('baz'===ev.columnNames[2]); + }else{ + f.counter = ev.rowNumber; + T.assert('number' === typeof ev.row); + } + }; + await wtest('exec',{ + sql: "select 'foo' foo, a bar, 'baz' baz from t limit 2", + callback: resultRowTest3, + columnNames: [], + rowMode: ':bar' + }, function(ev){ + log("exec() result row:",ev); + T.assert(2===resultRowTest3.counter); + }); + + await wtest('exec',{ + multi: true, + sql:[ + 'pragma foreign_keys=0;', + // ^^^ arbitrary query with no result columns + 'select a, b from t order by a desc; select a from t;' + // multi-exec only honors results from the first + // statement with result columns (regardless of whether) + // it has any rows). + ], + rowMode: 1, + resultRows: [] + },function(ev){ + const rows = ev.result.resultRows; + T.assert(3===rows.length). + assert(6===rows[0]); + }); + + await wtest('exec',{sql: 'delete from t where a>3'}); + + await wtest('exec',{ + sql: 'select count(a) from t', + resultRows: [] + },function(ev){ + ev = ev.result; + T.assert(1===ev.resultRows.length) + .assert(2===ev.resultRows[0][0]); + }); + + /***** close() tests must come last. *****/ + await wtest('close',{unlink:true},function(ev){ + T.assert(!promiserConfig.dbId); + T.assert('string' === typeof ev.result.filename); + }); + + await wtest('close', (ev)=>{ + T.assert(undefined === ev.result.filename); + }).finally(()=>logHtml('',"That's all, folks!")); + }/*runTests2()*/; + + + log("Init complete, but async init bits may still be running."); +})(); diff --git a/ext/wasm/testing1.html b/ext/wasm/testing1.html index 0c6447022..24c1e8236 100644 --- a/ext/wasm/testing1.html +++ b/ext/wasm/testing1.html @@ -27,7 +27,7 @@ <div>Most stuff on this page happens in the dev console.</div> <hr> <div id='test-output'></div> - <script src="api/sqlite3.js"></script> + <script src="sqlite3.js"></script> <script src="common/SqliteTestUtil.js"></script> <script src="testing1.js"></script> </body> diff --git a/ext/wasm/testing1.js b/ext/wasm/testing1.js index a733156e7..414421952 100644 --- a/ext/wasm/testing1.js +++ b/ext/wasm/testing1.js @@ -19,7 +19,8 @@ const toss = function(...args){throw new Error(args.join(' '))}; const debug = console.debug.bind(console); const eOutput = document.querySelector('#test-output'); - const log = console.log.bind(console) + const log = console.log.bind(console), + warn = console.warn.bind(console); const logHtml = function(...args){ log.apply(this, args); const ln = document.createElement('div'); @@ -162,10 +163,10 @@ } try { - throw new capi.WasmAllocError; + throw new sqlite3.WasmAllocError; }catch(e){ T.assert(e instanceof Error) - .assert(e instanceof capi.WasmAllocError); + .assert(e instanceof sqlite3.WasmAllocError); } try { @@ -1012,7 +1013,7 @@ wasm = capi.wasm; log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); log("Build options:",wasm.compileOptionUsed()); - + capi.sqlite3_web_persistent_dir()/*will install OPFS if available, plus a and non-locking VFS*/; if(1){ /* Let's grab those last few lines of test coverage for sqlite3-api.js... */ @@ -1067,13 +1068,7 @@ log('capi.wasm.exports',capi.wasm.exports); }; - sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ - /** Use a timeout so that we are (hopefully) out from under - the module init stack when our setup gets run. Just on - principle, not because we _need_ to be. */ - //console.debug("theModule =",theModule); - //setTimeout(()=>runTests(theModule), 0); - // ^^^ Chrome warns: "VIOLATION: setTimeout() handler took A WHOLE 50ms!" + self.sqlite3TestModule.initSqlite3().then(function(theModule){ self._MODULE = theModule /* this is only to facilitate testing from the console */ runTests(theModule); }); diff --git a/ext/wasm/testing2.html b/ext/wasm/testing2.html index 739c7f66b..25d5a9f5d 100644 --- a/ext/wasm/testing2.html +++ b/ext/wasm/testing2.html @@ -6,10 +6,10 @@ <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> <link rel="stylesheet" href="common/emscripten.css"/> <link rel="stylesheet" href="common/testing.css"/> - <title>sqlite3-worker.js tests</title> + <title>sqlite3-worker1.js tests</title> </head> <body> - <header id='titlebar'><span>sqlite3-worker.js tests</span></header> + <header id='titlebar'><span>sqlite3-worker1.js tests</span></header> <!-- emscripten bits --> <figure id="module-spinner"> <div class="spinner"></div> diff --git a/ext/wasm/testing2.js b/ext/wasm/testing2.js index 3a279513f..0a31c470a 100644 --- a/ext/wasm/testing2.js +++ b/ext/wasm/testing2.js @@ -10,17 +10,17 @@ *********************************************************************** - A basic test script for sqlite3-worker.js. + A basic test script for sqlite3-worker1.js. */ 'use strict'; (function(){ const T = self.SqliteTestUtil; - const SW = new Worker("api/sqlite3-worker.js"); + const SW = new Worker("sqlite3-worker1.js"); const DbState = { id: undefined }; const eOutput = document.querySelector('#test-output'); - const log = console.log.bind(console) + const log = console.log.bind(console); const logHtml = function(cssClass,...args){ log.apply(this, args); const ln = document.createElement('div'); @@ -31,24 +31,13 @@ const warn = console.warn.bind(console); const error = console.error.bind(console); const toss = (...args)=>{throw new Error(args.join(' '))}; - /** Posts a worker message as {type:type, data:data}. */ - const wMsg = function(type,data){ - log("Posting message to worker dbId="+(DbState.id||'default')+':',data); - SW.postMessage({ - type, - dbId: DbState.id, - data, - departureTime: performance.now() - }); - return SW; - }; SW.onerror = function(event){ error("onerror",event); }; let startTime; - + /** A queue for callbacks which are to be run in response to async DB commands. See the notes in runTests() for why we need @@ -74,28 +63,37 @@ logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); }; - const logEventResult = function(evd){ + const logEventResult = function(ev){ + const evd = ev.result; logHtml(evd.errorClass ? 'error' : '', - "runOneTest",evd.messageId,"Worker time =", - (evd.workerRespondTime - evd.workerReceivedTime),"ms.", + "runOneTest",ev.messageId,"Worker time =", + (ev.workerRespondTime - ev.workerReceivedTime),"ms.", "Round-trip event time =", - (performance.now() - evd.departureTime),"ms.", - (evd.errorClass ? evd.message : "") + (performance.now() - ev.departureTime),"ms.", + (evd.errorClass ? ev.message : "")//, JSON.stringify(evd) ); }; - const runOneTest = function(eventType, eventData, callback){ - T.assert(eventData && 'object'===typeof eventData); + const runOneTest = function(eventType, eventArgs, callback){ + T.assert(eventArgs && 'object'===typeof eventArgs); /* ^^^ that is for the testing and messageId-related code, not a hard requirement of all of the Worker-exposed APIs. */ - eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ - logEventResult(ev.data); + const messageId = MsgHandlerQueue.push(eventType,function(ev){ + logEventResult(ev); if(callback instanceof Function){ callback(ev); testCount(); } }); - wMsg(eventType, eventData); + const msg = { + type: eventType, + args: eventArgs, + dbId: DbState.id, + messageId: messageId, + departureTime: performance.now() + }; + log("Posting",eventType,"message to worker dbId="+(DbState.id||'default')+':',msg); + SW.postMessage(msg); }; /** Methods which map directly to onmessage() event.type keys. @@ -103,23 +101,31 @@ const dbMsgHandler = { open: function(ev){ DbState.id = ev.dbId; - log("open result",ev.data); + log("open result",ev); }, exec: function(ev){ - log("exec result",ev.data); + log("exec result",ev); }, export: function(ev){ - log("export result",ev.data); + log("export result",ev); }, error: function(ev){ - error("ERROR from the worker:",ev.data); - logEventResult(ev.data); + error("ERROR from the worker:",ev); + logEventResult(ev); }, resultRowTest1: function f(ev){ if(undefined === f.counter) f.counter = 0; - if(ev.data) ++f.counter; - //log("exec() result row:",ev.data); - T.assert(null===ev.data || 'number' === typeof ev.data.b); + if(null === ev.rowNumber){ + /* End of result set. */ + T.assert(undefined === ev.row) + .assert(Array.isArray(ev.columnNames)) + .assert(ev.columnNames.length); + }else{ + T.assert(ev.rowNumber > 0); + ++f.counter; + } + //log("exec() result row:",ev); + T.assert(null === ev.rowNumber || 'number' === typeof ev.row.b); } }; @@ -143,33 +149,33 @@ throw new Error("This is not supposed to be reached."); }; runOneTest('exec',{ - sql: ["create table t(a,b)", + sql: ["create table t(a,b);", "insert into t(a,b) values(1,2),(3,4),(5,6)" - ].join(';'), - multi: true, + ], resultRows: [], columnNames: [] }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(0===ev.resultRows.length) .assert(0===ev.columnNames.length); }); runOneTest('exec',{ sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], + resultRows: [], columnNames: [], saveSql:[] }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(3===ev.resultRows.length) .assert(1===ev.resultRows[0][0]) .assert(6===ev.resultRows[2][1]) .assert(2===ev.columnNames.length) .assert('b'===ev.columnNames[1]); }); + //if(1){ error("Returning prematurely for testing."); return; } runOneTest('exec',{ sql: 'select a a, b b from t order by a', resultRows: [], columnNames: [], rowMode: 'object' }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(3===ev.resultRows.length) .assert(1===ev.resultRows[0].a) .assert(6===ev.resultRows[2].b) @@ -181,7 +187,7 @@ resultRows: [], //rowMode: 'array', // array is the default in the Worker interface }, function(ev){ - ev = ev.data; + ev = ev.result; T.assert(1 === ev.resultRows.length) .assert(1 === ev.resultRows[0][0]); }); @@ -194,19 +200,19 @@ dbMsgHandler.resultRowTest1.counter = 0; }); runOneTest('exec',{ - multi: true, sql:[ - 'pragma foreign_keys=0;', + "pragma foreign_keys=0;", // ^^^ arbitrary query with no result columns - 'select a, b from t order by a desc; select a from t;' - // multi-exec only honors results from the first + "select a, b from t order by a desc;", + "select a from t;" + // multi-statement exec only honors results from the first // statement with result columns (regardless of whether) // it has any rows). ], rowMode: 1, resultRows: [] },function(ev){ - const rows = ev.data.resultRows; + const rows = ev.result.resultRows; T.assert(3===rows.length). assert(6===rows[0]); }); @@ -215,14 +221,14 @@ sql: 'select count(a) from t', resultRows: [] },function(ev){ - ev = ev.data; + ev = ev.result; T.assert(1===ev.resultRows.length) .assert(2===ev.resultRows[0][0]); }); if(0){ // export requires reimpl. for portability reasons. runOneTest('export',{}, function(ev){ - ev = ev.data; + ev = ev.result; T.assert('string' === typeof ev.filename) .assert(ev.buffer instanceof Uint8Array) .assert(ev.buffer.length > 1024) @@ -231,11 +237,11 @@ } /***** close() tests must come last. *****/ runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; + ev = ev.result; T.assert('string' === typeof ev.filename); }); runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; + ev = ev.result; T.assert(undefined === ev.filename); }); }; @@ -261,16 +267,8 @@ will fail and we have no way of cancelling them once they've been posted to the worker. - We currently do (2) because (A) it's certainly the most - client-friendly thing to do and (B) it seems likely that most - apps using this API will only have a single db to work with so - won't need to juggle multiple DB ids. If we revert to (1) then - the following call to runTests2() needs to be moved into the - callback function of the runOneTest() check for the 'open' - command. Note, also, that using approach (2) does not keep the - user from instead using approach (1), noting that doing so - requires explicit handling of the 'open' message to account for - it. + Which approach we use below depends on the boolean value of + waitForOpen. */ const waitForOpen = 1, simulateOpenError = 0 /* if true, the remaining tests will @@ -284,11 +282,11 @@ filename:'testing2.sqlite3', simulateError: simulateOpenError }, function(ev){ - //log("open result",ev); - T.assert('testing2.sqlite3'===ev.data.filename) - .assert(ev.data.dbId) - .assert(ev.data.messageId); - DbState.id = ev.data.dbId; + log("open result",ev); + T.assert('testing2.sqlite3'===ev.result.filename) + .assert(ev.dbId) + .assert(ev.messageId); + DbState.id = ev.dbId; if(waitForOpen) setTimeout(runTests2, 0); }); if(!waitForOpen) runTests2(); @@ -301,7 +299,7 @@ } ev = ev.data/*expecting a nested object*/; //log("main window onmessage:",ev); - if(ev.data && ev.data.messageId){ + if(ev.result && ev.messageId){ /* We're expecting a queued-up callback handler. */ const f = MsgHandlerQueue.shift(); if('error'===ev.type){ @@ -314,8 +312,8 @@ } switch(ev.type){ case 'sqlite3-api': - switch(ev.data){ - case 'worker-ready': + switch(ev.result){ + case 'worker1-ready': log("Message:",ev); self.sqlite3TestModule.setStatus(null); runTests(); @@ -1,9 +1,9 @@ -C Fix\suninitialized\svariable\sin\srollback-journal\sprocessing\sin\sos_kv.c -D 2022-09-12T15:59:35.676 +C Merge\skv-vfs\sbranch\sinto\sfiddle-opfs\sbranch\sto\sadd\skvvfs-based\swasm\sbuild\sand\sdemo. +D 2022-09-12T16:09:50.625 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in ee179f405fd5f8845473f888517c4ada46099306c33ae1f27dd1aef53fe8e867 +F Makefile.in 50e421194df031f669667fdb238c54959ecbea5a0b97dd3ed776cffbeea926d5 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -472,43 +472,62 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 +F ext/wasm/EXPORTED_FUNCTIONS.fiddle db7a4602f043cf4a5e4135be3609a487f9f1c83f05778bfbdf93766be4541b96 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile 12a672ab9125dc860457c2853f7651b98517e424d7a0e9714c89b28c5ff73800 -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 +F ext/wasm/GNUmakefile 6e642a0dc7ac43d9287e8f31c80ead469ddc7475a6d4ab7ac3b1feefcd4f7279 +F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 1dfd067b3cbd9a49cb204097367cf2f8fe71b5a3b245d9d82a24779fd4ac2394 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md b6d0fb64bfdf7bf9ce6938ea4104228f6f5bbef600f5d910b2f8c8694195988c +F ext/wasm/api/README.md d876597edd2b9542b6ea031adaaff1c042076fde7b670b1dc6d8a87b28a6631b F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b -F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 -F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b -F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb -F ext/wasm/api/sqlite3-api-worker.js 1124f404ecdf3c14d9f829425cef778cd683911a9883f0809a463c3c7773c9fd +F ext/wasm/api/sqlite3-api-cleanup.js 101919ec261644e2f6f0a59952fd9612127b69ea99b493277b2789ea478f9b6b +F ext/wasm/api/sqlite3-api-glue.js 2bf536a38cde324cf352bc2c575f8e22c6d204d667c0eda5a254ba45318914bc +F ext/wasm/api/sqlite3-api-oo1.js a9d8892be246548a9978ace506d108954aa13eb5ce25332975c8377953804ff3 +F ext/wasm/api/sqlite3-api-opfs.js 011799db398157cbd254264b6ebae00d7234b93d0e9e810345f213a5774993c0 +F ext/wasm/api/sqlite3-api-prologue.js 9e37ce4dfd74926d0df80dd7e72e33085db4bcee48e2c21236039be416a7dff2 +F ext/wasm/api/sqlite3-api-worker1.js d33062afa045fd4be01ba4abc266801807472558b862b30056211b00c9c347b4 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 8585793ca8311c7a0618b7e00ed2b3729799c20664a51f196258576e3d475c9e -F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 +F ext/wasm/api/sqlite3-wasm.c bf4637cf28463cada4b25f09651943c7ece004b253ef39b7ab68eaa60662aa09 +F ext/wasm/batch-runner.html 23209ade7981acce7ecd79d6eff9f4c5a4e8b14ae867ac27cd89b230be640fa6 +F ext/wasm/batch-runner.js 2abd146d3e3a66128ac0a2cc39bfd01e9811c9511fa10ec927d6649795f1ee50 +F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 -F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/common/testing.css 3a5143699c2b73a85b962271e1a9b3241b30d90e30d895e4f55665e648572962 +F ext/wasm/common/whwasmutil.js f7282ef36c9625330d4e6e82d1beec6678cd101e95e7108cd85db587a788c145 +F ext/wasm/demo-oo1.html 75646855b38405d82781246fd08c852a2b3bee05dd9f0fe10ab655a8cffb79aa +F ext/wasm/demo-oo1.js 477f230cce3455e701431436d892d8c6bfea2bdf1ddcdd32a273e2f4bb339801 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 -F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 +F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686 +F ext/wasm/index.html 5876ae0442bef5b37fec9a45ee0722798e47ef727723ada742d33554845afa6a +F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/kvvfs.make 7cc9cf10e744c3ba523c3eaf5c4af47028f3a5bb76db304ea8044a9b2a9d496f -F ext/wasm/kvvfs1.html 2acb241a6110a4ec581adbf07a23d5fc2ef9c7142aa9d60856732a102abc5016 -F ext/wasm/kvvfs1.js 46afaf4faba041bf938355627bc529854295e561f49db3a240c914e75a529338 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 -F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943 -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc +F ext/wasm/kvvfs.make dba616578bf91a76370a46494dd68a09c6dff5beb6d5561e2db65a27216e9630 +F ext/wasm/kvvfs1.html b8304cd5c7e7ec32c3b15521a95c322d6efdb8d22b3c4156123545dc54e07583 +F ext/wasm/kvvfs1.js a5075f98ffecd7d32348697db991fc61342d89aa20651034d1572af61890fb8b +F ext/wasm/scratchpad-opfs-main.html 4565cf194e66188190d35f70e82553e2e2d72b9809b73c94ab67b8cfd14d2e0c +F ext/wasm/scratchpad-opfs-main.js 69e960e9161f6412fd0c30f355d4112f1894d6609eb431e2d16d207d1380518e +F ext/wasm/scratchpad-opfs-worker.html 66c1d15d678f3bd306373d76b61c6c8aef988f61f4a8dd40185d452f9c6d2bf5 +F ext/wasm/scratchpad-opfs-worker.js 3ec2868c669713145c76eb5877c64a1b20741f741817b87c907a154b676283a9 +F ext/wasm/scratchpad-opfs-worker2.js 5f2237427ac537b8580b1c659ff14ad2621d1694043eaaf41ae18dbfef2e48c0 +F ext/wasm/speedtest1-worker.html 6b5fda04d0b69e8c2651689356cb0c28fd33aa1a82b03dcbc8b0d68fbd7ed57f +F ext/wasm/speedtest1-worker.js 356b9953add4449acf199793db9b76b11ee016021918d8daffd19f08ec68d305 +F ext/wasm/speedtest1.html 8f61cbe68300acca25dd9fa74dce79b774786e2b4feeb9bcbc46e1cefbfa6262 +F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x +F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 +F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 +F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9 +F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e +F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 +F ext/wasm/testing-worker1-promiser.js c62b5879339eef0b21aebd9d75bc125c86530edc17470afff18077f931cb704a +F ext/wasm/testing1.html 528001c7e32ee567abc195aa071fd9820cc3c8ffc9c8a39a75e680db05f0c409 +F ext/wasm/testing1.js 2def7a86c52ff28b145cb86188d5c7a49d5993f9b78c50d140e1c31551220955 +F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 +F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -574,7 +593,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_kv.c 2b4c04a470c05fe95a84d2ba3a5eb4874f0dbaa12da3a47f221ee3beec7eeda0 +F src/os_kv.c a188e92dac693b1c1b512d93b0c4dc85c1baad11e322b01121f87057996e4d11 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6 F src/os_unix.c d6322b78130d995160bb9cfb7850678ad6838b08c1d13915461b33326a406c04 F src/os_win.c e9454cb141908e8eef2102180bad353a36480612d5b736e4c2bd5777d9b25a34 @@ -1484,7 +1503,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c 8bf7ebac9ac316feed6656951249db531dc380c73fb3e3b22e224ffda96beff6 +F test/speedtest1.c 4001e0fcdbe5f136829319b547771d4a0d9b069cdd2d5d878222bed5d61e0b56 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -2004,8 +2023,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 250a935aeb94d3fadec0d3fe22de85de4e658e2fdb3be3aa9a8bbc8f7b7d8414 -R b0d44b8764d44e0d25567b7049186ee6 -U drh -Z b805cc7ee3bf0477637e784d78058472 +P 4e6ce329872eb733ba2f7f7879747c52761ae97790fd8ed169a25a79854cc3d9 e49682c5eac91958f143e639c5656ca54560d14f5805d514bf4aa0c206e63844 +R f15733833cf5e73dce86b4365f218581 +U stephan +Z f2a01bf4c99986993a2e00b39e93be73 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 383958616..bde2102af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e49682c5eac91958f143e639c5656ca54560d14f5805d514bf4aa0c206e63844
\ No newline at end of file +a7d8b26acd3c1ae344369e4d70804c0cab45272c0983cfd32d616a0a7b28acb9
\ No newline at end of file diff --git a/src/os_kv.c b/src/os_kv.c index 9260852db..a14dc5c54 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -583,7 +583,7 @@ static void kvvfsDecodeJournal( const char *zTxt, /* Text encoding. Zero-terminated */ int nTxt /* Bytes in zTxt, excluding zero terminator */ ){ - unsigned int n = 0; + unsigned int n; int c, i, mult; i = 0; mult = 1; diff --git a/test/speedtest1.c b/test/speedtest1.c index b115a57e2..ac0ab361a 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -7,6 +7,7 @@ static const char zHelp[] = "Usage: %s [--options] DATABASE\n" "Options:\n" " --autovacuum Enable AUTOVACUUM mode\n" + " --big-transactions Add BEGIN/END around large tests which normally don't\n" " --cachesize N Set the cache size to N\n" " --checkpoint Run PRAGMA wal_checkpoint after each test case\n" " --exclusive Enable locking_mode=EXCLUSIVE\n" @@ -20,6 +21,7 @@ static const char zHelp[] = " --mmap SZ MMAP the first SZ bytes of the database file\n" " --multithread Set multithreaded mode\n" " --nomemstat Disable memory statistics\n" + " --nomutex Open db with SQLITE_OPEN_NOMUTEX\n" " --nosync Set PRAGMA synchronous=OFF\n" " --notnull Add NOT NULL constraints to table columns\n" " --output FILE Store SQL output in FILE\n" @@ -97,6 +99,7 @@ static struct Global { int nRepeat; /* Repeat selects this many times */ int doCheckpoint; /* Run PRAGMA wal_checkpoint after each trans */ int nReserve; /* Reserve bytes */ + int doBigTransactions; /* Enable transactions on tests 410 and 510 */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ const char *zPK; /* Might be UNIQUE or PRIMARY KEY */ @@ -372,10 +375,12 @@ int speedtest1_numbername(unsigned int n, char *zOut, int nOut){ #define NAMEWIDTH 60 static const char zDots[] = "......................................................................."; +static int iTestNumber = 0; /* Current test # for begin/end_test(). */ void speedtest1_begin_test(int iTestNum, const char *zTestName, ...){ int n = (int)strlen(zTestName); char *zName; va_list ap; + iTestNumber = iTestNum; va_start(ap, zTestName); zName = sqlite3_vmprintf(zTestName, ap); va_end(ap); @@ -384,6 +389,11 @@ void speedtest1_begin_test(int iTestNum, const char *zTestName, ...){ zName[NAMEWIDTH] = 0; n = NAMEWIDTH; } + if( g.pScript ){ + fprintf(g.pScript,"-- begin test %d %.*s\n", iTestNumber, n, zName) + /* maintenance reminder: ^^^ code in ext/wasm expects %d to be + ** field #4 (as in: cut -d' ' -f4). */; + } if( g.bSqlOnly ){ printf("/* %4d - %s%.*s */\n", iTestNum, zName, NAMEWIDTH-n, zDots); }else{ @@ -404,6 +414,10 @@ void speedtest1_exec(const char*,...); void speedtest1_end_test(void){ sqlite3_int64 iElapseTime = speedtest1_timestamp() - g.iStart; if( g.doCheckpoint ) speedtest1_exec("PRAGMA wal_checkpoint;"); + assert( iTestNumber > 0 ); + if( g.pScript ){ + fprintf(g.pScript,"-- end test %d\n", iTestNumber); + } if( !g.bSqlOnly ){ g.iTotal += iElapseTime; printf("%4d.%03ds\n", (int)(iElapseTime/1000), (int)(iElapseTime%1000)); @@ -412,6 +426,7 @@ void speedtest1_end_test(void){ sqlite3_finalize(g.pStmt); g.pStmt = 0; } + iTestNumber = 0; } /* Report end of testing */ @@ -1105,12 +1120,24 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); speedtest1_begin_test(410, "%d SELECTS on an IPK", n); + if( g.doBigTransactions ){ + /* Historical note: tests 410 and 510 have historically not used + ** explicit transactions. The --big-transactions flag was added + ** 2022-09-08 to support the WASM/OPFS build, as the run-times + ** approach 1 minute for each of these tests if they're not in an + ** explicit transaction. The run-time effect of --big-transaciions + ** on native builds is negligible. */ + speedtest1_exec("BEGIN"); + } speedtest1_prepare("SELECT b FROM t5 WHERE a=?1; -- %d times",n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); sqlite3_bind_int(g.pStmt, 1, (sqlite3_int64)x1); speedtest1_run(); } + if( g.doBigTransactions ){ + speedtest1_exec("COMMIT"); + } speedtest1_end_test(); sz = n = g.szTest*700; @@ -1132,6 +1159,10 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); speedtest1_begin_test(510, "%d SELECTS on a TEXT PK", n); + if( g.doBigTransactions ){ + /* See notes for test 410. */ + speedtest1_exec("BEGIN"); + } speedtest1_prepare("SELECT b FROM t6 WHERE a=?1; -- %d times",n); for(i=1; i<=n; i++){ x1 = swizzle(i,maxb); @@ -1139,6 +1170,9 @@ void testset_main(void){ sqlite3_bind_text(g.pStmt, 1, zNum, -1, SQLITE_STATIC); speedtest1_run(); } + if( g.doBigTransactions ){ + speedtest1_exec("COMMIT"); + } speedtest1_end_test(); speedtest1_begin_test(520, "%d SELECT DISTINCT", n); speedtest1_exec("SELECT DISTINCT b FROM t5;"); @@ -2180,6 +2214,8 @@ int main(int argc, char **argv){ int nThread = 0; /* --threads value */ int mmapSize = 0; /* How big of a memory map to use */ int memDb = 0; /* --memdb. Use an in-memory database */ + int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + ; /* SQLITE_OPEN_xxx flags. */ char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ @@ -2192,6 +2228,12 @@ int main(int argc, char **argv){ int i; /* Loop counter */ int rc; /* API return code */ +#ifdef SQLITE_SPEEDTEST1_WASM + /* Resetting all state is important for the WASM build, which may + ** call main() multiple times. */ + memset(&g, 0, sizeof(g)); + iTestNumber = 0; +#endif #ifdef SQLITE_CKSUMVFS_STATIC sqlite3_register_cksumvfs(0); #endif @@ -2212,6 +2254,8 @@ int main(int argc, char **argv){ do{ z++; }while( z[0]=='-' ); if( strcmp(z,"autovacuum")==0 ){ doAutovac = 1; + }else if( strcmp(z,"big-transactions")==0 ){ + g.doBigTransactions = 1; }else if( strcmp(z,"cachesize")==0 ){ if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); i++; @@ -2254,6 +2298,8 @@ int main(int argc, char **argv){ if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); mmapSize = integerValue(argv[++i]); #endif + }else if( strcmp(z,"nomutex")==0 ){ + openFlags |= SQLITE_OPEN_NOMUTEX; }else if( strcmp(z,"nosync")==0 ){ noSync = 1; }else if( strcmp(z,"notnull")==0 ){ @@ -2395,7 +2441,8 @@ int main(int argc, char **argv){ sqlite3_initialize(); /* Open the database and the input file */ - if( sqlite3_open(memDb ? ":memory:" : zDbName, &g.db) ){ + if( sqlite3_open_v2(memDb ? ":memory:" : zDbName, &g.db, + openFlags, 0) ){ fatal_error("Cannot open database file: %s\n", zDbName); } #if SQLITE_VERSION_NUMBER>=3006001 |