diff options
author | stephan <stephan@noemail.net> | 2022-09-28 13:01:49 +0000 |
---|---|---|
committer | stephan <stephan@noemail.net> | 2022-09-28 13:01:49 +0000 |
commit | 5ad3631915e7f96f4647a7ca81c9b9e1d7c9375d (patch) | |
tree | 5d4070d3331783799b4d97199b6693fac884c548 /ext/wasm/batch-runner.js | |
parent | d98011852d956758475d158a1ad2e1992ff3a647 (diff) | |
download | sqlite-5ad3631915e7f96f4647a7ca81c9b9e1d7c9375d.tar.gz sqlite-5ad3631915e7f96f4647a7ca81c9b9e1d7c9375d.zip |
Correct duplicate copies of sqlite3-api.js being embedded in the wasmfs-based builds.
FossilOrigin-Name: bbfcfba260f39a9c91e82d87e06b1c2cb297c03498b4530aa3e7e01ef9916012
Diffstat (limited to 'ext/wasm/batch-runner.js')
-rw-r--r-- | ext/wasm/batch-runner.js | 531 |
1 files changed, 340 insertions, 191 deletions
diff --git a/ext/wasm/batch-runner.js b/ext/wasm/batch-runner.js index 24e28f6ea..7b671c9ea 100644 --- a/ext/wasm/batch-runner.js +++ b/ext/wasm/batch-runner.js @@ -19,30 +19,110 @@ const warn = console.warn.bind(console); let sqlite3; + /** Throws if the given sqlite3 result code is not 0. */ + const checkSqliteRc = (dbh,rc)=>{ + if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh)); + }; + + const sqlToDrop = [ + "SELECT type,name FROM sqlite_schema ", + "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ", + "AND name NOT LIKE '\\_%' escape '\\'" + ].join(''); + + const clearDbWebSQL = function(db){ + db.handle.transaction(function(tx){ + const onErr = (e)=>console.error(e); + const callback = function(tx, result){ + const rows = result.rows; + let i, n; + i = n = rows.length; + while(i--){ + const row = rows.item(i); + const name = JSON.stringify(row.name); + const type = row.type; + switch(type){ + case 'table': case 'view': case 'trigger':{ + const sql2 = 'DROP '+type+' '+name; + warn(db.id,':',sql2); + tx.executeSql(sql2, [], atEnd, onErr); + break; + } + default: + warn("Unhandled db entry type:",type,name); + break; + } + } + }; + tx.executeSql(sqlToDrop, [], callback, onErr); + db.handle.changeVersion(db.handle.version, "", ()=>{}, onErr, ()=>{}); + }); + }; + + const clearDbSqlite = function(db){ + // This would be SO much easier with the oo1 API, but we specifically want to + // inject metrics we can't get via that API, and we cannot reliably (OPFS) + // open the same DB twice to clear it using that API, so... + let pStmt = 0, pSqlBegin; + const capi = sqlite3.capi, wasm = capi.wasm; + const scope = wasm.scopedAllocPush(); + try { + const toDrop = []; + const ppStmt = wasm.scopedAllocPtr(); + let rc = capi.sqlite3_prepare_v2(db.handle, sqlToDrop, -1, ppStmt, null); + checkSqliteRc(db.handle,rc); + pStmt = wasm.getPtrValue(ppStmt); + while(capi.SQLITE_ROW===capi.sqlite3_step(pStmt)){ + toDrop.push(capi.sqlite3_column_text(pStmt,0), + capi.sqlite3_column_text(pStmt,1)); + } + capi.sqlite3_finalize(pStmt); + pStmt = 0; + while(toDrop.length){ + const name = toDrop.pop(); + const type = toDrop.pop(); + let sql2; + switch(type){ + case 'table': case 'view': case 'trigger': + sql2 = 'DROP '+type+' '+name; + break; + default: + warn("Unhandled db entry type:",type,name); + continue; + } + wasm.setPtrValue(ppStmt, 0); + warn(db.id,':',sql2); + rc = capi.sqlite3_prepare_v2(db.handle, sql2, -1, ppStmt, null); + checkSqliteRc(db.handle,rc); + pStmt = wasm.getPtrValue(ppStmt); + capi.sqlite3_step(pStmt); + capi.sqlite3_finalize(pStmt); + pStmt = 0; + } + }finally{ + if(pStmt) capi.sqlite3_finalize(pStmt); + wasm.scopedAllocPop(scope); + } + }; + + + const E = (s)=>document.querySelector(s); const App = { e: { - output: document.querySelector('#test-output'), - selSql: document.querySelector('#sql-select'), - btnRun: document.querySelector('#sql-run'), - btnRunNext: document.querySelector('#sql-run-next'), - btnRunRemaining: document.querySelector('#sql-run-remaining'), - btnExportMetrics: document.querySelector('#export-metrics'), - btnClear: document.querySelector('#output-clear'), - btnReset: document.querySelector('#db-reset'), - cbReverseLog: document.querySelector('#cb-reverse-log-order') + output: E('#test-output'), + selSql: E('#sql-select'), + btnRun: E('#sql-run'), + btnRunNext: E('#sql-run-next'), + btnRunRemaining: E('#sql-run-remaining'), + btnExportMetrics: E('#export-metrics'), + btnClear: E('#output-clear'), + btnReset: E('#db-reset'), + cbReverseLog: E('#cb-reverse-log-order'), + selImpl: E('#select-impl') }, db: Object.create(null), + dbs: Object.create(null), cache:{}, - metrics:{ - /** - Map of sql-file to timing metrics. We currently only store - the most recent run of each file, but we really should store - all runs so that we can average out certain values which vary - significantly across runs. e.g. a mandelbrot-generating query - will have a wide range of runtimes when run 10 times in a - row. - */ - }, log: console.log.bind(console), warn: console.warn.bind(console), cls: function(){this.e.output.innerHTML = ''}, @@ -62,47 +142,159 @@ if(1) this.logHtml2('error', ...args); }, - openDb: function(fn, unlinkFirst=true){ - if(this.db.ptr){ - toss("Already have an opened db."); - } + execSql: async function(name,sql){ + const db = this.getSelectedDb(); + const banner = "========================================"; + this.logHtml(banner, + "Running",name,'('+sql.length,'bytes) using',db.id); const capi = this.sqlite3.capi, wasm = capi.wasm; + let pStmt = 0, pSqlBegin; const stack = wasm.scopedAllocPush(); - let pDb = 0; - try{ - if(unlinkFirst && fn){ - if(':'!==fn[0]) capi.wasm.sqlite3_wasm_vfs_unlink(fn); - this.clearStorage(); - } - const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; - const ppDb = wasm.scopedAllocPtr(); - const rc = capi.sqlite3_open_v2(fn, ppDb, oFlags, null); - pDb = wasm.getPtrValue(ppDb) - if(rc){ - if(pDb) capi.sqlite3_close_v2(pDb); - toss("sqlite3_open_v2() failed with code",rc); - } - }finally{ - wasm.scopedAllocPop(stack); + const metrics = db.metrics = Object.create(null); + metrics.prepTotal = metrics.stepTotal = 0; + metrics.stmtCount = 0; + metrics.malloc = 0; + metrics.strcpy = 0; + this.blockControls(true); + if(this.gotErr){ + this.logErr("Cannot run SQL: error cleanup is pending."); + return; } - this.db.filename = fn; - this.db.ptr = pDb; - this.logHtml("Opened db:",fn); - return this.db.ptr; - }, + // Run this async so that the UI can be updated for the above header... + const dumpMetrics = ()=>{ + metrics.evalSqlEnd = performance.now(); + metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart); + this.logHtml(db.id,"metrics:");//,JSON.stringify(metrics, undefined, ' ')); + this.logHtml("prepare() count:",metrics.stmtCount); + this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms", + "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())"); + this.logHtml("Time in step():",metrics.stepTotal,"ms", + "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())"); + this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms"); + this.logHtml("Overhead (time - prep - step):", + (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); + this.logHtml(banner,"End of",name); + }; - closeDb: function(unlink=false){ - if(this.db.ptr){ - this.sqlite3.capi.sqlite3_close_v2(this.db.ptr); - this.logHtml("Closed db",this.db.filename); - if(unlink){ - capi.wasm.sqlite3_wasm_vfs_unlink(this.db.filename); - this.clearStorage(); - } - this.db.ptr = this.db.filename = undefined; + let runner; + if('websql'===db.id){ + runner = function(resolve, reject){ + /* WebSQL cannot execute multiple statements, nor can it execute SQL without + an explicit transaction. Thus we have to do some fragile surgery on the + input SQL. Since we're only expecting carefully curated inputs, the hope is + that this will suffice. */ + metrics.evalSqlStart = performance.now(); + const sqls = sql.split(/;+\n/); + const rxBegin = /^BEGIN/i, rxCommit = /^COMMIT/i, rxComment = /^\s*--/; + try { + const nextSql = ()=>{ + let x = sqls.shift(); + while(x && rxComment.test(x)) x = sqls.shift(); + return x && x.trim(); + }; + const transaction = function(tx){ + let s; + for(;s = nextSql(); s = s.nextSql()){ + if(rxBegin.test(s)) continue; + else if(rxCommit.test(s)) break; + ++metrics.stmtCount; + const t = performance.now(); + tx.executeSql(s); + metrics.stepTotal += performance.now() - t; + } + }; + while(sqls.length){ + db.handle.transaction(transaction); + } + resolve(this); + }catch(e){ + this.gotErr = e; + reject(e); + return; + } + }.bind(this); + }else{/*sqlite3 db...*/ + runner = function(resolve, reject){ + metrics.evalSqlStart = performance.now(); + try { + let t; + let sqlByteLen = sql.byteLength; + const [ppStmt, pzTail] = wasm.scopedAllocPtr(2); + t = performance.now(); + pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed"); + metrics.malloc = performance.now() - t; + metrics.byteLength = sqlByteLen; + let pSql = pSqlBegin; + const pSqlEnd = pSqlBegin + sqlByteLen; + t = performance.now(); + wasm.heap8().set(sql, pSql); + wasm.setMemValue(pSql + sqlByteLen, 0); + metrics.strcpy = performance.now() - t; + let breaker = 0; + while(pSql && wasm.getMemValue(pSql,'i8')){ + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + t = performance.now(); + let rc = capi.sqlite3_prepare_v3( + db.handle, pSql, sqlByteLen, 0, ppStmt, pzTail + ); + metrics.prepTotal += performance.now() - t; + checkSqliteRc(db.handle, rc); + pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue/*empty statement*/; + ++metrics.stmtCount; + t = performance.now(); + rc = capi.sqlite3_step(pStmt); + capi.sqlite3_finalize(pStmt); + pStmt = 0; + metrics.stepTotal += performance.now() - t; + switch(rc){ + case capi.SQLITE_ROW: + case capi.SQLITE_DONE: break; + default: checkSqliteRc(db.handle, rc); toss("Not reached."); + } + } + resolve(this); + }catch(e){ + if(pStmt) capi.sqlite3_finalize(pStmt); + this.gotErr = e; + reject(e); + return; + }finally{ + wasm.scopedAllocPop(stack); + } + }.bind(this); + } + let p; + if(1){ + p = new Promise(function(res,rej){ + setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/; + }); + }else{ + p = new Promise(runner); + } + return p.catch( + (e)=>this.logErr("Error via execSql("+name+",...):",e.message) + ).finally(()=>{ + dumpMetrics(); + this.blockControls(false); + }); + }, + + clearDb: function(){ + const db = this.getSelectedDb(); + if('websql'===db.id){ + this.logErr("TODO: clear websql db."); + return; } + if(!db.handle) return; + const capi = this.sqlite3, wasm = capi.wasm; + //const scope = wasm.scopedAllocPush( + this.logErr("TODO: clear db"); }, - + /** Loads batch-runner.list and populates the selection list from it. Returns a promise which resolves to nothing in particular @@ -123,7 +315,7 @@ } if(!r.ok) toss("Loading",infile,"failed:",r.statusText); txt = await r.text(); - const warning = document.querySelector('#warn-list'); + const warning = E('#warn-list'); if(warning) warning.remove(); }catch(e){ this.logErr(e.message); @@ -169,13 +361,6 @@ return sql; }/*fetchFile()*/, - /** Throws if the given sqlite3 result code is not 0. */ - checkRc: function(rc){ - if(this.db.ptr && rc){ - toss("Prepare failed:",this.sqlite3.capi.sqlite3_errmsg(this.db.ptr)); - } - }, - /** Disable or enable certain UI controls. */ blockControls: function(disable){ document.querySelectorAll('.disable-during-eval').forEach((e)=>e.disabled = disable); @@ -191,11 +376,11 @@ const rc = []; Object.keys(this.metrics).sort().forEach((k)=>{ const m = this.metrics[k]; - delete m.evalFileStart; - delete m.evalFileEnd; + delete m.evalSqlStart; + delete m.evalSqlEnd; const mk = Object.keys(m).sort(); if(!rc.length){ - rc.push(['file', ...mk]); + rc.push(['db', ...mk]); } const row = [k.split('/').pop()/*remove dir prefix from filename*/]; rc.push(row); @@ -205,6 +390,10 @@ }, metricsToBlob: function(colSeparator='\t'){ + if(1){ + this.logErr("TODO: re-do metrics dump"); + return; + } const ar = [], ma = this.metricsToArrays(); if(!ma.length){ this.logErr("Metrics are empty. Run something."); @@ -239,114 +428,84 @@ */ evalFile: async function(fn){ const sql = await this.fetchFile(fn); - const banner = "========================================"; - this.logHtml(banner, - "Running",fn,'('+sql.length,'bytes)...'); - const capi = this.sqlite3.capi, wasm = capi.wasm; - let pStmt = 0, pSqlBegin; - const stack = wasm.scopedAllocPush(); - const metrics = this.metrics[fn] = Object.create(null); - metrics.prepTotal = metrics.stepTotal = 0; - metrics.stmtCount = 0; - metrics.malloc = 0; - metrics.strcpy = 0; - this.blockControls(true); - if(this.gotErr){ - this.logErr("Cannot run ["+fn+"]: error cleanup is pending."); - return; + return this.execSql(fn,sql); + }/*evalFile()*/, + + /** + Clears all DB tables in all _opened_ databases. Because of + disparities between backends, we cannot simply "unlink" the + databases to clean them up. + */ + clearStorage: function(onlySelectedDb=false){ + const list = onlySelectDb + ? [('boolean'===typeof onlySelectDb) + ? this.dbs[this.e.selImpl.value] + : onlySelectDb] + : Object.values(this.dbs); + for(let db of list){ + if(db && db.handle){ + this.logHtml("Clearing db",db.id); + d.clear(); + } } - // Run this async so that the UI can be updated for the above header... - const ff = function(resolve, reject){ - metrics.evalFileStart = performance.now(); - try { - let t; - let sqlByteLen = sql.byteLength; - const [ppStmt, pzTail] = wasm.scopedAllocPtr(2); - t = performance.now(); - pSqlBegin = wasm.alloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed"); - metrics.malloc = performance.now() - t; - metrics.byteLength = sqlByteLen; - let pSql = pSqlBegin; - const pSqlEnd = pSqlBegin + sqlByteLen; - t = performance.now(); - wasm.heap8().set(sql, pSql); - wasm.setMemValue(pSql + sqlByteLen, 0); - metrics.strcpy = performance.now() - t; - let breaker = 0; - while(pSql && wasm.getMemValue(pSql,'i8')){ - wasm.setPtrValue(ppStmt, 0); - wasm.setPtrValue(pzTail, 0); - t = performance.now(); - let rc = capi.sqlite3_prepare_v3( - this.db.ptr, pSql, sqlByteLen, 0, ppStmt, pzTail - ); - metrics.prepTotal += performance.now() - t; - this.checkRc(rc); - pStmt = wasm.getPtrValue(ppStmt); - pSql = wasm.getPtrValue(pzTail); - sqlByteLen = pSqlEnd - pSql; - if(!pStmt) continue/*empty statement*/; - ++metrics.stmtCount; - t = performance.now(); - rc = capi.sqlite3_step(pStmt); - capi.sqlite3_finalize(pStmt); - pStmt = 0; - metrics.stepTotal += performance.now() - t; - switch(rc){ - case capi.SQLITE_ROW: - case capi.SQLITE_DONE: break; - default: this.checkRc(rc); toss("Not reached."); - } + }, + + /** + Fetches the handle of the db associated with + this.e.selImpl.value, opening it if needed. + */ + getSelectedDb: function(){ + if(!this.dbs.memdb){ + for(let opt of this.e.selImpl.options){ + const d = this.dbs[opt.value] = Object.create(null); + d.id = opt.value; + switch(d.id){ + case 'virtualfs': + d.filename = 'file:/virtualfs.sqlite3?vfs=unix-none'; + break; + case 'memdb': + d.filename = ':memory:'; + break; + case 'wasmfs-opfs': + d.filename = 'file:'+(this.sqlite3.capi.sqlite3_wasmfs_opfs_dir())+'/wasmfs-opfs.sqlite3'; + break; + case 'websql': + d.filename = 'websql.db'; + break; + default: + this.logErr("Unhandled db selection option (see details in the console).",opt); + toss("Unhandled db init option"); } + } + }/*first-time init*/ + const dbId = this.e.selImpl.value; + const d = this.dbs[dbId]; + if(d.handle) return d; + if('websql' === dbId){ + d.handle = self.openDatabase('batch-runner', '0.1', 'foo', 1024 * 1024 * 50); + d.clear = ()=>clearDbWebSQL(d); + }else{ + const capi = this.sqlite3.capi, wasm = capi.wasm; + const stack = wasm.scopedAllocPush(); + let pDb = 0; + try{ + const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + const ppDb = wasm.scopedAllocPtr(); + const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, null); + pDb = wasm.getPtrValue(ppDb) + if(rc) toss("sqlite3_open_v2() failed with code",rc); }catch(e){ - if(pStmt) capi.sqlite3_finalize(pStmt); - this.gotErr = e; - //throw e; - reject(e); - return; + if(pDb) capi.sqlite3_close_v2(pDb); }finally{ - wasm.dealloc(pSqlBegin); wasm.scopedAllocPop(stack); - this.blockControls(false); } - metrics.evalFileEnd = performance.now(); - metrics.evalTimeTotal = (metrics.evalFileEnd - metrics.evalFileStart); - this.logHtml("Metrics:");//,JSON.stringify(metrics, undefined, ' ')); - this.logHtml("prepare() count:",metrics.stmtCount); - this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms", - "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())"); - this.logHtml("Time in step():",metrics.stepTotal,"ms", - "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())"); - this.logHtml("Total runtime:",metrics.evalTimeTotal,"ms"); - this.logHtml("Overhead (time - prep - step):", - (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); - this.logHtml(banner,"End of",fn); - resolve(this); - }.bind(this); - let p; - if(1){ - p = new Promise(function(res,rej){ - setTimeout(()=>ff(res, rej), 50)/*give UI a chance to output the "running" banner*/; - }); - }else{ - p = new Promise(ff); - } - return p.catch((e)=>this.logErr("Error via evalFile("+fn+"):",e.message)); - }/*evalFile()*/, - - clearStorage: function(){ - const sz = sqlite3.capi.sqlite3_web_kvvfs_size(); - const n = sqlite3.capi.sqlite3_web_kvvfs_clear(this.db.filename || ''); - this.logHtml("Cleared kvvfs local/sessionStorage:", - n,"entries totaling approximately",sz,"bytes."); - }, - - resetDb: function(){ - if(this.db.ptr){ - const fn = this.db.filename; - this.closeDb(true); - this.openDb(fn,false); + d.handle = pDb; + d.clear = ()=>clearDbSqlite(d); } + d.clear(); + this.logHtml("Opened db:",dbId); + console.log("db =",d); + return d; }, run: function(sqlite3){ @@ -356,38 +515,24 @@ this.logHtml("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); this.logHtml("WASM heap size =",wasm.heap8().length); this.loadSqlList(); - let pDir, dbFile; - if(sqlite3.capi.sqlite3_vfs_find('kvvfs')){ - dbFile = 1 ? 'local' : 'session'; - this.logHtml("Using KVVFS storage:",dbFile); + if(capi.sqlite3_wasmfs_opfs_dir()){ + E('#warn-opfs').classList.remove('hidden'); }else{ - pDir = capi.sqlite3_wasmfs_opfs_dir(); - if(pDir){ - dbFile = pDir+"/speedtest.db"; - this.logHtml("Using persistent storage:",dbFile); - }else{ - dbFile = ':memory:'; - this.logHtml("Using",dbFile,"storage."); - } + E('#warn-opfs').remove(); + E('option[value=wasmfs-opfs]').disabled = true; } - if(!pDir){ - document.querySelector('#warn-opfs').remove(); + if('function' === typeof self.openDatabase){ + E('#warn-websql').classList.remove('hidden'); + }else{ + E('option[value=websql]').disabled = true; + E('#warn-websql').remove(); } - this.openDb(dbFile, true); const who = this; - const eReverseLogNotice = document.querySelector('#reverse-log-notice'); if(this.e.cbReverseLog.checked){ - eReverseLogNotice.classList.remove('hidden'); this.e.output.classList.add('reverse'); } this.e.cbReverseLog.addEventListener('change', function(){ - if(this.checked){ - who.e.output.classList.add('reverse'); - eReverseLogNotice.classList.remove('hidden'); - }else{ - who.e.output.classList.remove('reverse'); - eReverseLogNotice.classList.add('hidden'); - } + who.e.output.classList[this.checked ? 'add' : 'remove']('reverse'); }, false); this.e.btnClear.addEventListener('click', ()=>this.cls(), false); this.e.btnRun.addEventListener('click', function(){ @@ -400,7 +545,7 @@ who.evalFile(who.e.selSql.value); }, false); this.e.btnReset.addEventListener('click', function(){ - who.resetDb(); + who.clearStorage(true); }, false); this.e.btnExportMetrics.addEventListener('click', function(){ who.logHtml2('warning',"Triggering download of metrics CSV. Check your downloads folder."); @@ -408,6 +553,9 @@ //const m = who.metricsToArrays(); //console.log("Metrics:",who.metrics, m); }); + this.e.selImpl.addEventListener('change', function(){ + who.getSelectedDb(); + }); this.e.btnRunRemaining.addEventListener('click', async function(){ let v = who.e.selSql.value; const timeStart = performance.now(); @@ -430,6 +578,7 @@ self.sqlite3TestModule.initSqlite3().then(function(theEmccModule){ self._MODULE = theEmccModule /* this is only to facilitate testing from the console */; sqlite3 = theEmccModule.sqlite3; + console.log("App",App); App.run(theEmccModule.sqlite3); }); })(); |