diff options
author | drh <> | 2024-09-14 16:39:02 +0000 |
---|---|---|
committer | drh <> | 2024-09-14 16:39:02 +0000 |
commit | dbff02abc5b4fd9457e7952d714c018386e23cd5 (patch) | |
tree | 0b3b523da99b4908fc5b340e4114bc4029b6122e | |
parent | 99f50dd21949a8fb60a71537447b4ffd1fc0306a (diff) | |
parent | 3d56d59adc1e0c60ca752e2dd062e78d3927102e (diff) | |
download | sqlite-dbff02abc5b4fd9457e7952d714c018386e23cd5.tar.gz sqlite-dbff02abc5b4fd9457e7952d714c018386e23cd5.zip |
Enhancements to sqlite_dbpage() so that it accepts INSERT statements that can
extend or truncate the database. Add the sqlite3-rsync utility program that
make a copy of a live database over SSH.
FossilOrigin-Name: b7a8ce4c8c5fc6a3b4744d412d96f99d2452eb4086ad84472511da3b4d6afec6
-rw-r--r-- | Makefile.in | 15 | ||||
-rw-r--r-- | Makefile.msc | 14 | ||||
-rw-r--r-- | ext/misc/sha1.c | 34 | ||||
-rw-r--r-- | main.mk | 14 | ||||
-rw-r--r-- | manifest | 25 | ||||
-rw-r--r-- | manifest.uuid | 2 | ||||
-rw-r--r-- | src/dbpage.c | 67 | ||||
-rw-r--r-- | src/shell.c.in | 2 | ||||
-rw-r--r-- | tool/sqlite3-rsync.c | 1845 |
9 files changed, 1985 insertions, 33 deletions
diff --git a/Makefile.in b/Makefile.in index e359975d8..c9e27a4cc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -700,6 +700,20 @@ sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -USQLITE_THREADSAFE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(TEXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) + scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) @@ -1191,6 +1205,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ diff --git a/Makefile.msc b/Makefile.msc index 41e5c1081..295e07b07 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1867,6 +1867,19 @@ sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\con dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +RSYNC_SRC = \ + $(TOP)\tool\sqlite3-rsync.c \ + $(SQLITE3C) + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS) + $(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) + scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2317,6 +2330,7 @@ SHELL_DEP = \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ + $(TOP)\ext\misc\sha1.c \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\sqlar.c \ $(TOP)\ext\misc\uint.c \ diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 9790a1d87..07d797060 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -196,7 +196,8 @@ static void hash_step_vformat( ** zOut[]. zOut[] must be at least 41 bytes long. */ static void hash_finish( SHA1Context *p, /* The SHA1 context to finish and render */ - char *zOut /* Store hexadecimal hash here */ + char *zOut, /* Store hex or binary hash here */ + int bAsBinary /* 1 for binary hash, 0 for hex hash */ ){ unsigned int i; unsigned char finalcount[8]; @@ -215,11 +216,15 @@ static void hash_finish( for (i = 0; i < 20; i++){ digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); } - for(i=0; i<20; i++){ - zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; - zOut[i*2+1] = zEncode[digest[i] & 0xf]; + if( bAsBinary ){ + memcpy(zOut, digest, 20); + }else{ + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; } - zOut[i*2]= 0; } /* End of the hashing logic *****************************************************************************/ @@ -251,8 +256,13 @@ static void sha1Func( }else{ hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } - hash_finish(&cx, zOut); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); + if( sqlite3_user_data(context)!=0 ){ + hash_finish(&cx, zOut, 1); + sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); + }else{ + hash_finish(&cx, zOut, 0); + sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + } } /* @@ -365,7 +375,7 @@ static void sha1QueryFunc( } sqlite3_finalize(pStmt); } - hash_finish(&cx, zOut); + hash_finish(&cx, zOut, 0); sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } @@ -379,11 +389,17 @@ int sqlite3_sha_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; + static int one = 1; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha1Func, 0, 0); + 0, sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1b", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + (void*)&one, sha1Func, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8|SQLITE_DIRECTONLY, 0, @@ -568,6 +568,19 @@ dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(EXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) + scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) @@ -768,6 +781,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ @@ -1,11 +1,11 @@ -C Close\sthe\sdb\sas\ssqlite3_analyzer\sexits.\s(See\s[forum:af384c2315d9718a|forum\spost,\s"sqlite3_analyzer\snot\sclosing\sWAL-mode\sdb\scleanly"\s].) -D 2024-09-13T16:10:04.337 +C Enhancements\sto\ssqlite_dbpage()\sso\sthat\sit\saccepts\sINSERT\sstatements\sthat\scan\nextend\sor\struncate\sthe\sdatabase.\s\sAdd\sthe\ssqlite3-rsync\sutility\sprogram\sthat\nmake\sa\scopy\sof\sa\slive\sdatabase\sover\sSSH. +D 2024-09-14T16:39:02.179 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 7753650b4204e3ccd55a4e6a0d73a5a01f737dcefb099d901ce1de5df9d0b82c +F Makefile.in 31368ad3e1800bb5f311adede543ee456ca7d2595403c7f131797ae65a7d415c F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 6e8925dca6dc8c3e9cce042bbf347d20164653e63aeafcf6f6a28e27cf976d8b +F Makefile.msc 62ace0005c53b52f189c20c1d6d8fa4dbd2a37c90d9c1362b60f4fb4c841fa15 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -420,7 +420,7 @@ F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 -F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d +F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 391342c3c0907f57bbb9ab60ce4b3cfe1ea61161996b449033984673d18980fd +F main.mk 8b9c0252aef57b5b2a10f34b8b46e89f9ed06bdccef1df98673a12f34e9b3e79 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -714,7 +714,7 @@ F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d49 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c b224d3db0f28c4a5f1407c50107a0a8133bd244ff3c7f6f8cedeb896a8cf1b64 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a -F src/dbpage.c f8c93e845d1093554247c1e757cb443fc48ffbcb112cecfdebeca4b6aa6e5c6e +F src/dbpage.c 12e49515d67d4a59625d71f9aa42499556cfdc2e4f1ea49086e674a7f47f46e5 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 @@ -768,7 +768,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 470db843788d74234cc1e6873ac51c0ae6529994a52146fefe2e77c0754cbf96 +F src/shell.c.in 375f8a183126be96ec73db4e42c57917ff10a0900846b1b722dd4f8cef537812 F src/sqlite.h.in 77f55bd1978a04a14db211732f0a609077cf60ba4ccf9baf39988f508945419c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2174,6 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a +F tool/sqlite3-rsync.c f34c43d88141dff62fb83067059bca9ec79d85fbc5be92bfe0d95db95ce21107 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2212,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4cad385b90eaca2d90e3375e473472145af4134160b81097a8535d06638c2e4a -R 506f0349cf71d4b56754bbc21843c2e9 -U larrybr -Z 8cfad2cb033fbf4c7a29251fd8b9a990 +P 94ceac98845e31249b656dcdb8a58f456b9212dc83968ea333852a66d72a0dae 9961334c8007e7cb6ae55885075b74acddc4fa701b359cf67e0f3c237d7eba4a +R 35bd9e1a3f1cc54ea6c1adcfd7040bcd +U drh +Z 2f1609bf1d6f611498fb654c8834250d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0888cebb1..4facd3596 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94ceac98845e31249b656dcdb8a58f456b9212dc83968ea333852a66d72a0dae +b7a8ce4c8c5fc6a3b4744d412d96f99d2452eb4086ad84472511da3b4d6afec6 diff --git a/src/dbpage.c b/src/dbpage.c index 9740b418a..42b24f9b8 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -28,7 +28,13 @@ ** ** The data field of sqlite_dbpage table can be updated. The new ** value must be a BLOB which is the correct page size, otherwise the -** update fails. Rows may not be deleted or inserted. +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -51,6 +57,8 @@ struct DbpageCursor { struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ + int nTrunc; /* Entries in aTrunc[] */ + Pgno *aTrunc; /* Truncation size for each database */ }; /* Columns */ @@ -59,7 +67,6 @@ struct DbpageTable { #define DBPAGE_COLUMN_SCHEMA 2 - /* ** Connect to or create a dbpagevfs virtual table. */ @@ -100,6 +107,8 @@ static int dbpageConnect( ** Disconnect from or destroy a dbpagevfs virtual table. */ static int dbpageDisconnect(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3_free(pTab->aTrunc); sqlite3_free(pVtab); return SQLITE_OK; } @@ -325,6 +334,7 @@ static int dbpageUpdate( Btree *pBt; Pager *pPager; int szPage; + int isInsert; (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ @@ -337,18 +347,20 @@ static int dbpageUpdate( } if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ pgno = (Pgno)sqlite3_value_int(argv[2]); + isInsert = 1; }else{ pgno = sqlite3_value_int(argv[0]); if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ zErr = "cannot insert"; goto update_fail; } + isInsert = 0; } if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ iDb = 0; }else{ const char *zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; + iDb = sqlite3FindDbName(pTab->db, zSchema); if( iDb<0 ){ zErr = "no such schema"; goto update_fail; @@ -363,18 +375,31 @@ static int dbpageUpdate( if( sqlite3_value_type(argv[3])!=SQLITE_BLOB || sqlite3_value_bytes(argv[3])!=szPage ){ - zErr = "bad page value"; - goto update_fail; + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert ){ + if( iDb>=pTab->nTrunc ){ + testcase( pTab->aTrunc!=0 ); + pTab->aTrunc = sqlite3_realloc(pTab->aTrunc, (iDb+1)*sizeof(Pgno)); + if( pTab->aTrunc ){ + int j; + for(j=pTab->nTrunc; j<iDb; j++) pTab->aTrunc[j] = 0; + pTab->nTrunc = iDb+1; + }else{ + return SQLITE_NOMEM; + } + } + pTab->aTrunc[iDb] = pgno; + }else{ + zErr = "bad page value"; + goto update_fail; + } } pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ const void *pData = sqlite3_value_blob(argv[3]); - assert( pData!=0 || pTab->db->mallocFailed ); - if( pData - && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK - ){ - memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -398,6 +423,26 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ Btree *pBt = db->aDb[i].pBt; if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); } + if( pTab->nTrunc>0 ){ + memset(pTab->aTrunc, 0, sizeof(pTab->aTrunc[0])*pTab->nTrunc); + } + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + int iDb; + DbpageTable *pTab = (DbpageTable *)pVtab; + + for(iDb=0; iDb<pTab->nTrunc; iDb++){ + if( pTab->aTrunc[iDb]>0 ){ + Btree *pBt = pTab->db->aDb[iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerTruncateImage(pPager, pTab->aTrunc[iDb]); + pTab->aTrunc[iDb] = 0; + } + } return SQLITE_OK; } @@ -422,7 +467,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ dbpageBegin, /* xBegin */ - 0, /* xSync */ + dbpageSync, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ diff --git a/src/shell.c.in b/src/shell.c.in index 9bc6c2566..2b0e506ed 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1209,6 +1209,7 @@ INCLUDE test_windirent.c INCLUDE ../ext/misc/memtrace.c INCLUDE ../ext/misc/pcachetrace.c INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/sha1.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c INCLUDE ../ext/misc/percentile.c @@ -5392,6 +5393,7 @@ static void open_db(ShellState *p, int openFlags){ #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif + sqlite3_sha_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_uint_init(p->db, 0, 0); sqlite3_stmtrand_init(p->db, 0, 0); diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c new file mode 100644 index 000000000..2a58125aa --- /dev/null +++ b/tool/sqlite3-rsync.c @@ -0,0 +1,1845 @@ +/* +** 2024-09-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program that makes a copy of a live SQLite database +** using a bandwidth-efficient protocol, similar to "rsync". +*/ +#include <stdio.h> +#include <stdlib.h> +#include <ctype.h> +#include <string.h> +#include <stdarg.h> +#include "sqlite3.h" + +static const char zUsage[] = + "sqlite3-rsync ORIGIN REPLICA ?OPTIONS?\n" + "\n" + "One of ORIGIN or REPLICA is a pathname to a database on the local\n" + "machine and the other is of the form \"USER@HOST:PATH\" describing\n" + "a database on a remote machine. This utility makes REPLICA into a\n" + "copy of ORIGIN\n" + "\n" + "OPTIONS:\n" + "\n" + " --exe PATH Name of the sqlite3-rsync program on the remote side\n" + " --help Show this help screen\n" + " --ssh PATH Name of the SSH program used to reach the remote side\n" + " -v Verbose. Multiple v's for increasing output\n" + " --version Show detailed version information\n" +; + +typedef unsigned char u8; + +/* Context for the run */ +typedef struct SQLiteRsync SQLiteRsync; +struct SQLiteRsync { + const char *zOrigin; /* Name of the origin */ + const char *zReplica; /* Name of the replica */ + const char *zErrFile; /* Append error messages to this file */ + FILE *pOut; /* Transmit to the other side */ + FILE *pIn; /* Receive from the other side */ + FILE *pLog; /* Duplicate output here if not NULL */ + sqlite3 *db; /* Database connection */ + int nErr; /* Number of errors encountered */ + u8 eVerbose; /* Bigger for more output. 0 means none. */ + u8 bCommCheck; /* True to debug the communication protocol */ + u8 isRemote; /* On the remote side of a connection */ + u8 isReplica; /* True if running on the replica side */ + u8 iProtocol; /* Protocol version number */ + sqlite3_uint64 nOut; /* Bytes transmitted */ + sqlite3_uint64 nIn; /* Bytes received */ + unsigned int nPage; /* Total number of pages in the database */ + unsigned int szPage; /* Database page size */ + unsigned int nHashSent; /* Hashes sent (replica to origin) */ + unsigned int nPageSent; /* Page contents sent (origin to replica) */ +}; + +/* The version number of the protocol. Sent in the *_BEGIN message +** to verify that both sides speak the same dialect. +*/ +#define PROTOCOL_VERSION 1 + + +/* Magic numbers to identify particular messages sent over the wire. +*/ +#define ORIGIN_BEGIN 0x41 /* Initial message */ +#define ORIGIN_END 0x42 /* Time to quit */ +#define ORIGIN_ERROR 0x43 /* Error message from the remote */ +#define ORIGIN_PAGE 0x44 /* New page data */ +#define ORIGIN_TXN 0x45 /* Transaction commit */ +#define ORIGIN_MSG 0x46 /* Informational message */ + +#define REPLICA_BEGIN 0x61 /* Welcome message */ +#define REPLICA_ERROR 0x62 /* Error. Report and quit. */ +#define REPLICA_END 0x63 /* Replica wants to stop */ +#define REPLICA_HASH 0x64 /* One or more pages hashes to report */ +#define REPLICA_READY 0x65 /* Read to receive page content */ +#define REPLICA_MSG 0x66 /* Informational message */ + + +/**************************************************************************** +** Beginning of the popen2() implementation copied from Fossil ************* +****************************************************************************/ +#ifdef _WIN32 +#include <windows.h> +#include <fcntl.h> +/* +** Print a fatal error and quit. +*/ +static void win32_fatal_error(const char *zMsg){ + fprintf(stderr, "%s", zMsg); + exit(1); +} +extern int _open_osfhandle(intptr_t,int); +#else +#include <unistd.h> +#include <signal.h> +#include <sys/wait.h> +#endif + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** This code is copied out of SQLite. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define INT_TO_PTR(X) ((void*)(X)) +# define PTR_TO_INT(X) ((int)(X)) +#endif + +/* Register SQL functions provided by ext/misc/sha1.c */ +extern int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +); + +#ifdef _WIN32 +/* +** On windows, create a child process and specify the stdin, stdout, +** and stderr channels for that process to use. +** +** Return the number of errors. +*/ +static int win32_create_child_process( + wchar_t *zCmd, /* The command that the child process will run */ + HANDLE hIn, /* Standard input */ + HANDLE hOut, /* Standard output */ + HANDLE hErr, /* Standard error */ + DWORD *pChildPid /* OUT: Child process handle */ +){ + STARTUPINFOW si; + PROCESS_INFORMATION pi; + BOOL rc; + + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + SetHandleInformation(hIn, HANDLE_FLAG_INHERIT, TRUE); + si.hStdInput = hIn; + SetHandleInformation(hOut, HANDLE_FLAG_INHERIT, TRUE); + si.hStdOutput = hOut; + SetHandleInformation(hErr, HANDLE_FLAG_INHERIT, TRUE); + si.hStdError = hErr; + rc = CreateProcessW( + NULL, /* Application Name */ + zCmd, /* Command-line */ + NULL, /* Process attributes */ + NULL, /* Thread attributes */ + TRUE, /* Inherit Handles */ + 0, /* Create flags */ + NULL, /* Environment */ + NULL, /* Current directory */ + &si, /* Startup Info */ + &pi /* Process Info */ + ); + if( rc ){ + CloseHandle( pi.hProcess ); + CloseHandle( pi.hThread ); + *pChildPid = pi.dwProcessId; + }else{ + win32_fatal_error("cannot create child process"); + } + return rc!=0; +} +void *win32_utf8_to_unicode(const char *zUtf8){ + int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0); + wchar_t *zUnicode = malloc( nByte*2 ); + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte); + return zUnicode; +} +#endif + +/* +** Create a child process running shell command "zCmd". *ppOut is +** a FILE that becomes the standard input of the child process. +** (The caller writes to *ppOut in order to send text to the child.) +** *ppIn is stdout from the child process. (The caller +** reads from *ppIn in order to receive input from the child.) +** Note that *ppIn is an unbuffered file descriptor, not a FILE. +** The process ID of the child is written into *pChildPid. +** +** Return the number of errors. +*/ +static int popen2( + const char *zCmd, /* Command to run in the child process */ + FILE **ppIn, /* Read from child using this file descriptor */ + FILE **ppOut, /* Write to child using this file descriptor */ + int *pChildPid, /* PID of the child process */ + int bDirect /* 0: run zCmd as a shell cmd. 1: run directly */ +){ +#ifdef _WIN32 + HANDLE hStdinRd, hStdinWr, hStdoutRd, hStdoutWr, hStderr; + SECURITY_ATTRIBUTES saAttr; + DWORD childPid = 0; + int fd; + + saAttr.nLength = sizeof(saAttr); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + hStderr = GetStdHandle(STD_ERROR_HANDLE); + if( !CreatePipe(&hStdoutRd, &hStdoutWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdout"); + } + SetHandleInformation( hStdoutRd, HANDLE_FLAG_INHERIT, FALSE); + + if( !CreatePipe(&hStdinRd, &hStdinWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdin"); + } + SetHandleInformation( hStdinWr, HANDLE_FLAG_INHERIT, FALSE); + + win32_create_child_process(win32_utf8_to_unicode(zCmd), + hStdinRd, hStdoutWr, hStderr,&childPid); + *pChildPid = childPid; + fd = _open_osfhandle(PTR_TO_INT(hStdoutRd), 0); + *ppIn = fdopen(fd, "r"); + fd = _open_osfhandle(PTR_TO_INT(hStdinWr), 0); + *ppOut = _fdopen(fd, "w"); + CloseHandle(hStdinRd); + CloseHandle(hStdoutWr); + return 0; +#else + int pin[2], pout[2]; + *ppIn = 0; + *ppOut = 0; + *pChildPid = 0; + + if( pipe(pin)<0 ){ + return 1; + } + if( pipe(pout)<0 ){ + close(pin[0]); + close(pin[1]); + return 1; + } + *pChildPid = fork(); + if( *pChildPid<0 ){ + close(pin[0]); + close(pin[1]); + close(pout[0]); + close(pout[1]); + *pChildPid = 0; + return 1; + } + signal(SIGPIPE,SIG_IGN); + if( *pChildPid==0 ){ + int fd; + /* This is the child process */ + close(0); + fd = dup(pout[0]); + if( fd!=0 ) { + fprintf(stderr,"popen2() failed to open file descriptor 0"); + exit(1); + } + close(pout[0]); + close(pout[1]); + close(1); + fd = dup(pin[1]); + if( fd!=1 ){ + fprintf(stderr,"popen() failed to open file descriptor 1"); + exit(1); + } + close(pin[0]); + close(pin[1]); + if( bDirect ){ + execl(zCmd, zCmd, (char*)0); + }else{ + execl("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0); + } + return 1; + }else{ + /* This is the parent process */ + close(pin[1]); + *ppIn = fdopen(pin[0], "r"); + close(pout[0]); + *ppOut = fdopen(pout[1], "w"); + return 0; + } +#endif +} + +/* +** Close the connection to a child process previously created using +** popen2(). +*/ +static void pclose2(FILE *pIn, FILE *pOut, int childPid){ +#ifdef _WIN32 + /* Not implemented, yet */ + fclose(pIn); + fclose(pOut); +#else + fclose(pIn); + fclose(pOut); + while( waitpid(0, 0, WNOHANG)>0 ) {} +#endif +} +/***************************************************************************** +** End of the popen2() implementation copied from Fossil ********************* +*****************************************************************************/ + +/***************************************************************************** +** Beginning of the append_escaped_arg() routine, adapted from the Fossil ** +** subroutine nameed blob_append_escaped_arg() ** +*****************************************************************************/ +/* +** ASCII (for reference): +** x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf +** 0x ^` ^a ^b ^c ^d ^e ^f ^g \b \t \n () \f \r ^n ^o +** 1x ^p ^q ^r ^s ^t ^u ^v ^w ^x ^y ^z ^{ ^| ^} ^~ ^ +** 2x () ! " # $ % & ' ( ) * + , - . / +** 3x 0 1 2 3 4 5 6 7 8 9 : ; < = > ? +** 4x @ A B C D E F G H I J K L M N O +** 5x P Q R S T U V W X Y Z [ \ ] ^ _ +** 6x ` a b c d e f g h i j k l m n o +** 7x p q r s t u v w x y z { | } ~ ^_ +*/ + +/* +** Meanings for bytes in a filename: +** +** 0 Ordinary character. No encoding required +** 1 Needs to be escaped +** 2 Illegal character. Do not allow in a filename +** 3 First byte of a 2-byte UTF-8 +** 4 First byte of a 3-byte UTF-8 +** 5 First byte of a 4-byte UTF-8 +*/ +static const char aSafeChar[256] = { +#ifdef _WIN32 +/* Windows +** Prohibit: all control characters, including tab, \r and \n. +** Escape: (space) " # $ % & ' ( ) * ; < > ? [ ] ^ ` { | } +*/ +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 0x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 1x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, /* 3x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, /* 5x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, /* 7x */ +#else +/* Unix +** Prohibit: all control characters, including tab, \r and \n +** Escape: (space) ! " # $ % & ' ( ) * ; < > ? [ \ ] ^ ` { | } +*/ +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 0x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 1x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, /* 3x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, /* 5x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, /* 7x */ +#endif + /* all bytes 0x80 through 0xbf are unescaped, being secondary + ** bytes to UTF8 characters. Bytes 0xc0 through 0xff are the + ** first byte of a UTF8 character and do get escaped */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 8x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 9x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* ax */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* bx */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* cx */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* dx */ + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* ex */ + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 /* fx */ +}; + +/* +** pStr is a shell command under construction. This routine safely +** appends filename argument zIn. It returns 0 on success or non-zero +** on any error. +** +** The argument is escaped if it contains white space or other characters +** that need to be escaped for the shell. If zIn contains characters +** that cannot be safely escaped, then throw a fatal error. +** +** If the isFilename argument is true, then the argument is expected +** to be a filename. As shell commands commonly have command-line +** options that begin with "-" and since we do not want an attacker +** to be able to invoke these switches using filenames that begin +** with "-", if zIn begins with "-", prepend an additional "./" +** (or ".\\" on Windows). +*/ +int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ + int i; + unsigned char c; + int needEscape = 0; + int n = sqlite3_str_length(pStr); + char *z = sqlite3_str_value(pStr); + + /* Look for illegal byte-sequences and byte-sequences that require + ** escaping. No control-characters are allowed. All spaces and + ** non-ASCII unicode characters and some punctuation characters require + ** escaping. */ + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + if( aSafeChar[c] ){ + unsigned char x = aSafeChar[c]; + needEscape = 1; + if( x==2 ){ + /* Bad ASCII character */ + return 1; + }else if( x>2 ){ + if( (zIn[i+1]&0xc0)!=0x80 + || (x>=4 && (zIn[i+2]&0xc0)!=0x80) + || (x==5 && (zIn[i+3]&0xc0)!=0x80) + ){ + /* Bad UTF8 character */ + return 1; + } + i += x-2; + } + } + } + + /* Separate from the previous argument by a space */ + if( n>0 && !isspace(z[n-1]) ){ + sqlite3_str_appendchar(pStr, 1, ' '); + } + + /* Check for characters that need quoting */ + if( !needEscape ){ + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); +#if defined(_WIN32) + sqlite3_str_appendchar(pStr, 1, '\\'); +#else + sqlite3_str_appendchar(pStr, 1, '/'); +#endif + } + sqlite3_str_appendall(pStr, zIn); + }else{ +#if defined(_WIN32) + /* Quoting strategy for windows: + ** Put the entire name inside of "...". Any " characters within + ** the name get doubled. + */ + sqlite3_str_appendchar(pStr, 1, '"'); + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '\\'); + }else if( zIn[0]=='/' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + } + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + sqlite3_str_appendchar(pStr, 1, (char)c); + if( c=='"' ) sqlite3_str_appendchar(pStr, 1, '"'); + if( c=='\\' ) sqlite3_str_appendchar(pStr, 1, '\\'); + if( c=='%' && isFilename ) sqlite3_str_append(pStr, "%cd:~,%", 7); + } + sqlite3_str_appendchar(pStr, 1, '"'); +#else + /* Quoting strategy for unix: + ** If the name does not contain ', then surround the whole thing + ** with '...'. If there is one or more ' characters within the + ** name, then put \ before each special character. + */ + if( strchr(zIn,'\'') ){ + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '/'); + } + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + if( aSafeChar[c] && aSafeChar[c]!=2 ){ + sqlite3_str_appendchar(pStr, 1, '\\'); + } + sqlite3_str_appendchar(pStr, 1, (char)c); + } + }else{ + sqlite3_str_appendchar(pStr, 1, '\''); + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '/'); + } + sqlite3_str_appendall(pStr, zIn); + sqlite3_str_appendchar(pStr, 1, '\''); + } +#endif + } + return 0; +} +/***************************************************************************** +** End of the append_escaped_arg() routine, adapted from the Fossil ** +*****************************************************************************/ + +/***************************************************************************** +** The Hash Engine +** +** This is basically SHA3, though with a 160-bit hash, and reducing the +** number of rounds in the KeccakF1600 step function from 24 to 6. +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DHash_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef Hash_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define Hash_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define Hash_BYTEORDER 4321 +# else +# define Hash_BYTEORDER 0 +# endif +#endif + +typedef sqlite3_uint64 u64; + +/* +** State structure for a Hash hash in progress +*/ +typedef struct HashContext HashContext; +struct HashContext { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(HashContext *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<<x)|(a>>(64-x))) + + /* v---- Number of rounds. SHA3 has 24 here. */ + for(i=0; i<6; i++){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void HashInit(HashContext *p, int iSize){ + memset(p, 0, sizeof(*p)); + p->iSize = iSize; + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if Hash_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif Hash_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the HashUpdate function to add new content +** to the hash +*/ +static void HashUpdate( + HashContext *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; + if( aData==0 ) return; +#if Hash_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7<nData; i+=8){ + p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; i<nData; i++){ +#if Hash_BYTEORDER==1234 + p->u.x[p->nLoaded] ^= aData[i]; +#elif Hash_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke HashFinal() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *HashFinal(HashContext *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + HashUpdate(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + HashUpdate(p, &c2, 1); + p->nLoaded = p->nRate - 1; + HashUpdate(p, &c3, 1); + } + for(i=0; i<p->nRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Implementation of the hash(X) function. +** +** Return a 160-bit BLOB which is the hash of X. +*/ +static void hashFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + HashContext cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + if( eType==SQLITE_NULL ) return; + HashInit(&cx, 160); + if( eType==SQLITE_BLOB ){ + HashUpdate(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + HashUpdate(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, HashFinal(&cx), 160/8, SQLITE_TRANSIENT); +} + +/* Register the hash function */ +static int hashRegister(sqlite3 *db){ + return sqlite3_create_function(db, "hash", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, hashFunc, 0, 0); +} + +/* End of the hashing logic +*****************************************************************************/ + +/* +** Return the tail of a file pathname. The tail is the last component +** of the path. For example, the tail of "/a/b/c.d" is "c.d". +*/ +const char *file_tail(const char *z){ + const char *zTail = z; + if( !zTail ) return 0; + while( z[0] ){ + if( z[0]=='/' ) zTail = &z[1]; + z++; + } + return zTail; +} + +/* +** Append error message text to the error file, if an error file is +** specified. In any case, increment the error count. +*/ +static void logError(SQLiteRsync *p, const char *zFormat, ...){ + if( p->zErrFile ){ + FILE *pErr = fopen(p->zErrFile, "a"); + if( pErr ){ + va_list ap; + va_start(ap, zFormat); + vfprintf(pErr, zFormat, ap); + va_end(ap); + fclose(pErr); + } + } + p->nErr++; +} + + +/* Read a single big-endian 32-bit unsigned integer from the input +** stream. Return 0 on success and 1 if there are any errors. +*/ +static int readUint32(SQLiteRsync *p, unsigned int *pU){ + unsigned char buf[4]; + if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){ + *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + p->nIn += 4; + return 0; + }else{ + p->nErr++; + return 1; + } +} + +/* Write a single big-endian 32-bit unsigned integer to the output stream. +** Return 0 on success and 1 if there are any errors. +*/ +static int writeUint32(SQLiteRsync *p, unsigned int x){ + unsigned char buf[4]; + buf[3] = x & 0xff; + x >>= 8; + buf[2] = x & 0xff; + x >>= 8; + buf[1] = x & 0xff; + x >>= 8; + buf[0] = x; + if( p->pLog ) fwrite(buf, sizeof(buf), 1, p->pLog); + if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){ + logError(p, "failed to write 32-bit integer 0x%x", x); + return 1; + } + p->nOut += 4; + return 0; +} + +/* Read a single byte from the wire. +*/ +int readByte(SQLiteRsync *p){ + int c = fgetc(p->pIn); + if( c!=EOF ) p->nIn++; + return c; +} + +/* Write a single byte into the wire. +*/ +void writeByte(SQLiteRsync *p, int c){ + if( p->pLog ) fputc(c, p->pLog); + fputc(c, p->pOut); + p->nOut++; +} + +/* Read a power of two encoded as a single byte. +*/ +int readPow2(SQLiteRsync *p){ + int x = readByte(p); + if( x>=32 ){ + logError(p, "read invalid page size %d\n", x); + return 0; + } + return 1<<x; +} + +/* Write a power-of-two value onto the wire as a single byte. +*/ +void writePow2(SQLiteRsync *p, int c){ + int n; + if( c<0 || (c&(c-1))!=0 ){ + logError(p, "trying to read invalid page size %d\n", c); + } + for(n=0; c>1; n++){ c /= 2; } + writeByte(p, n); +} + +/* Read an array of bytes from the wire. +*/ +void readBytes(SQLiteRsync *p, int nByte, void *pData){ + if( fread(pData, 1, nByte, p->pIn)==nByte ){ + p->nIn += nByte; + }else{ + logError(p, "failed to read %d bytes", nByte); + } +} + +/* Write an array of bytes onto the wire. +*/ +void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ + if( p->pLog ) fwrite(pData, 1, nByte, p->pLog); + if( fwrite(pData, 1, nByte, p->pOut)==nByte ){ + p->nOut += nByte; + }else{ + logError(p, "failed to write %d bytes", nByte); + } +} + +/* Report an error. +** +** If this happens on the remote side, we send back a *_ERROR +** message. On the local side, the error message goes to stderr. +*/ +static void reportError(SQLiteRsync *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + unsigned int nMsg; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0; + if( p->isRemote ){ + if( p->isReplica ){ + putc(REPLICA_ERROR, p->pOut); + }else{ + putc(ORIGIN_ERROR, p->pOut); + } + writeUint32(p, nMsg); + writeBytes(p, nMsg, zMsg); + fflush(p->pOut); + }else{ + fprintf(stderr, "%s\n", zMsg); + } + logError(p, "%s\n", zMsg); + sqlite3_free(zMsg); +} + +/* Send an informational message. +** +** If this happens on the remote side, we send back a *_MSG +** message. On the local side, the message goes to stdout. +*/ +static void infoMsg(SQLiteRsync *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + unsigned int nMsg; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0; + if( p->isRemote ){ + if( p->isReplica ){ + putc(REPLICA_MSG, p->pOut); + }else{ + putc(ORIGIN_MSG, p->pOut); + } + writeUint32(p, nMsg); + writeBytes(p, nMsg, zMsg); + fflush(p->pOut); + }else{ + printf("%s\n", zMsg); + } + sqlite3_free(zMsg); +} + +/* Receive and report an error message coming from the other side. +*/ +static void readAndDisplayMessage(SQLiteRsync *p, int c){ + unsigned int n = 0; + char *zMsg; + const char *zPrefix; + if( c==ORIGIN_ERROR || c==REPLICA_ERROR ){ + zPrefix = "ERROR: "; + }else{ + zPrefix = ""; + } + readUint32(p, &n); + if( n==0 ){ + fprintf(stderr,"ERROR: unknown (possibly out-of-memory)\n"); + }else{ + zMsg = sqlite3_malloc64( n+1 ); + if( zMsg==0 ){ + fprintf(stderr, "ERROR: out-of-memory\n"); + return; + } + memset(zMsg, 0, n+1); + readBytes(p, n, zMsg); + fprintf(stderr,"%s%s\n", zPrefix, zMsg); + if( zPrefix[0] ) logError(p, "%s%s\n", zPrefix, zMsg); + sqlite3_free(zMsg); + } +} + +/* Construct a new prepared statement. Report an error and return NULL +** if anything goes wrong. +*/ +static sqlite3_stmt *prepareStmtVA( + SQLiteRsync *p, + char *zFormat, + va_list ap +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + char *zToFree = 0; + int rc; + + if( strchr(zFormat,'%') ){ + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ){ + reportError(p, "out-of-memory"); + return 0; + }else{ + zToFree = zSql; + } + }else{ + zSql = zFormat; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc || pStmt==0 ){ + reportError(p, "unable to prepare SQL [%s]: %s", zSql, + sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + pStmt = 0; + } + if( zToFree ) sqlite3_free(zToFree); + return pStmt; +} +static sqlite3_stmt *prepareStmt( + SQLiteRsync *p, + char *zFormat, + ... +){ + sqlite3_stmt *pStmt; + va_list ap; + va_start(ap, zFormat); + pStmt = prepareStmtVA(p, zFormat, ap); + va_end(ap); + return pStmt; +} + +/* Run a single SQL statement +*/ +static void runSql(SQLiteRsync *p, char *zSql, ...){ + sqlite3_stmt *pStmt; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + if( pStmt ){ + int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ) rc = sqlite3_step(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + } + sqlite3_finalize(pStmt); + } +} + +/* Run an SQL statement that returns a single unsigned 32-bit integer result +*/ +static int runSqlReturnUInt( + SQLiteRsync *p, + unsigned int *pRes, + char *zSql, + ... +){ + sqlite3_stmt *pStmt; + int res = 0; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + if( pStmt==0 ){ + res = 1; + }else{ + int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + *pRes = (unsigned int)(sqlite3_column_int64(pStmt, 0)&0xffffffff); + }else{ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + res = 1; + } + sqlite3_finalize(pStmt); + } + return res; +} + +/* Run an SQL statement that returns a single TEXT value that is no more +** than 99 bytes in length. +*/ +static int runSqlReturnText( + SQLiteRsync *p, + char *pRes, + char *zSql, + ... +){ + sqlite3_stmt *pStmt; + int res = 0; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + pRes[0] = 0; + if( pStmt==0 ){ + res = 1; + }else{ + int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + const unsigned char *a = sqlite3_column_text(pStmt, 0); + int n; + if( a==0 ){ + pRes[0] = 0; + }else{ + n = sqlite3_column_bytes(pStmt, 0); + if( n>99 ) n = 99; + memcpy(pRes, a, n); + pRes[n] = 0; + } + }else{ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + res = 1; + } + sqlite3_finalize(pStmt); + } + return res; +} + +/* Close the database connection associated with p +*/ +static void closeDb(SQLiteRsync *p){ + if( p->db ){ + sqlite3_close(p->db); + p->db = 0; + } +} + +/* +** Run the origin-side protocol. +** +** Begin by sending the ORIGIN_BEGIN message with two arguments, +** nPage, and szPage. Then enter a loop responding to message from +** the replica: +** +** REPLICA_ERROR size text +** +** Report an error from the replica and quit +** +** REPLICA_END +** +** The replica is terminating. Stop processing now. +** +** REPLICA_HASH hash +** +** The argument is the 20-byte SHA1 hash for the next page +** page hashes appear in sequential order with no gaps. +** +** REPLICA_READY +** +** The replica has sent all the hashes that it intends to send. +** This side (the origin) can now start responding with page +** content for pages that do not have a matching hash. +*/ +static void originSide(SQLiteRsync *p){ + int rc = 0; + int c = 0; + unsigned int nPage = 0; + unsigned int iPage = 0; + unsigned int szPg = 0; + sqlite3_stmt *pCkHash = 0; + char buf[200]; + + p->isReplica = 0; + if( p->bCommCheck ){ + infoMsg(p, "origin zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", + p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + writeByte(p, ORIGIN_END); + fflush(p->pOut); + }else{ + /* Open the ORIGIN database. */ + rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); + if( rc ){ + reportError(p, "cannot open origin \"%s\": %s", + p->zOrigin, sqlite3_errmsg(p->db)); + closeDb(p); + return; + } + hashRegister(p->db); + runSql(p, "BEGIN"); + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( sqlite3_stricmp(buf,"wal")!=0 ){ + reportError(p, "Origin database is not in WAL mode"); + } + runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); + runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); + + if( p->nErr==0 ){ + /* Send the ORIGIN_BEGIN message */ + writeByte(p, ORIGIN_BEGIN); + writeByte(p, PROTOCOL_VERSION); + writePow2(p, szPg); + writeUint32(p, nPage); + fflush(p->pOut); + p->nPage = nPage; + p->szPage = szPg; + p->iProtocol = PROTOCOL_VERSION; + } + } + + /* Respond to message from the replica */ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=REPLICA_END ){ + switch( c ){ + case REPLICA_BEGIN: { + /* This message is only sent if the replica received an origin-protocol + ** that is larger than what it knows about. The replica sends back + ** a counter-proposal of an earlier protocol which the origin can + ** accept by resending a new ORIGIN_BEGIN. */ + p->iProtocol = readByte(p); + writeByte(p, ORIGIN_BEGIN); + writeByte(p, p->iProtocol); + writePow2(p, p->szPage); + writeUint32(p, p->nPage); + break; + } + case REPLICA_MSG: + case REPLICA_ERROR: { + readAndDisplayMessage(p, c); + break; + } + case REPLICA_HASH: { + if( pCkHash==0 ){ + runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); + pCkHash = prepareStmt(p, + "INSERT INTO badHash SELECT pgno FROM sqlite_dbpage('main')" + " WHERE pgno=?1 AND hash(data)!=?2" + ); + if( pCkHash==0 ) break; + } + p->nHashSent++; + iPage++; + sqlite3_bind_int64(pCkHash, 1, iPage); + readBytes(p, 20, buf); + sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC); + rc = sqlite3_step(pCkHash); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pCkHash); + break; + } + case REPLICA_READY: { + sqlite3_stmt *pStmt; + int needPageOne = 0; + sqlite3_finalize(pCkHash); + pCkHash = 0; + if( iPage+1<p->nPage ){ + runSql(p, "WITH RECURSIVE c(n) AS" + " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + " INSERT INTO badHash SELECT n FROM c", + iPage+1, p->nPage); + } + pStmt = prepareStmt(p, + "SELECT pgno, data" + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); + const void *pContent = sqlite3_column_blob(pStmt, 1); + if( pgno==1 ){ + needPageOne = 1; + }else{ + writeByte(p, ORIGIN_PAGE); + writeUint32(p, (unsigned int)sqlite3_column_int64(pStmt, 0)); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + } + sqlite3_finalize(pStmt); + if( needPageOne ){ + pStmt = prepareStmt(p, + "SELECT data" + " FROM sqlite_dbpage('main')" + " WHERE pgno=1" + ); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const void *pContent = sqlite3_column_blob(pStmt, 0); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, 1); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + sqlite3_finalize(pStmt); + } + writeByte(p, ORIGIN_TXN); + writeUint32(p, nPage); + writeByte(p, ORIGIN_END); + goto origin_end; + } + default: { + reportError(p, "Unknown message 0x%02x %lld bytes into conversation", + c, p->nIn); + break; + } + } + } + +origin_end: + if( pCkHash ) sqlite3_finalize(pCkHash); + closeDb(p); +} + +/* +** Run the replica-side protocol. The protocol is passive in the sense +** that it only response to message from the origin side. +** +** ORIGIN_BEGIN idProtocol szPage nPage +** +** The origin is reporting the protocol version number, the size of +** each page in the origin database (sent as a single-byte power-of-2), +** and the number of pages in the origin database. +** This procedure checks compatibility, and if everything is ok, +** it starts sending hashes of pages already present back to the origin. +** +** ORIGIN_ERROR size text +** +** Report the received error and quit. +** +** ORIGIN_PAGE pgno content +** +** Update the content of the given page. +** +** ORIGIN_TXN pgno +** +** Close the update transaction. The total database size is pgno +** pages. +** +** ORIGIN_END +** +** Expect no more transmissions from the origin. +*/ +static void replicaSide(SQLiteRsync *p){ + int c; + sqlite3_stmt *pIns = 0; + unsigned int szOPage = 0; + char buf[65536]; + + p->isReplica = 1; + if( p->bCommCheck ){ + infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", + p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + writeByte(p, REPLICA_END); + fflush(p->pOut); + } + + /* Respond to message from the origin. The origin will initiate the + ** the conversation with an ORIGIN_BEGIN message. + */ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=ORIGIN_END ){ + switch( c ){ + case ORIGIN_MSG: + case ORIGIN_ERROR: { + readAndDisplayMessage(p, c); + break; + } + case ORIGIN_BEGIN: { + unsigned int nOPage = 0; + unsigned int nRPage = 0, szRPage = 0; + int rc = 0; + sqlite3_stmt *pStmt = 0; + + closeDb(p); + p->iProtocol = readByte(p); + szOPage = readPow2(p); + readUint32(p, &nOPage); + if( p->nErr ) break; + if( p->iProtocol>PROTOCOL_VERSION ){ + /* If the protocol version on the origin side is larger, send back + ** a REPLICA_BEGIN message with the protocol version number of the + ** replica side. This gives the origin an opportunity to resend + ** a new ORIGIN_BEGIN with a reduced protocol version. */ + writeByte(p, REPLICA_BEGIN); + writeByte(p, PROTOCOL_VERSION); + break; + } + p->nPage = nOPage; + p->szPage = szOPage; + rc = sqlite3_open(p->zReplica, &p->db); + if( rc ){ + reportError(p, "cannot open replica \"%s\": %s", + p->zReplica, sqlite3_errmsg(p->db)); + closeDb(p); + break; + } + hashRegister(p->db); + if( runSqlReturnUInt(p, &nRPage, "PRAGMA page_count") ){ + break; + } + if( nRPage==0 ){ + runSql(p, "PRAGMA page_size=%u", szOPage); + runSql(p, "PRAGMA journal_mode=WAL"); + runSql(p, "SELECT * FROM sqlite_schema"); + } + runSql(p, "BEGIN IMMEDIATE"); + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( strcmp(buf, "wal")!=0 ){ + reportError(p, "replica is not in WAL mode"); + break; + } + runSqlReturnUInt(p, &nRPage, "PRAGMA page_count"); + runSqlReturnUInt(p, &szRPage, "PRAGMA page_size"); + if( szRPage!=szOPage ){ + reportError(p, "page size mismatch; origin is %d bytes and " + "replica is %d bytes", szOPage, szRPage); + break; + } + + pStmt = prepareStmt(p, + "SELECT hash(data) FROM sqlite_dbpage" + " WHERE pgno<=min(%d,%d)" + " ORDER BY pgno", nRPage, nOPage); + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const unsigned char *a = sqlite3_column_blob(pStmt, 0); + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + p->nHashSent++; + } + sqlite3_finalize(pStmt); + writeByte(p, REPLICA_READY); + fflush(p->pOut); + runSql(p, "PRAGMA writable_schema=ON"); + break; + } + case ORIGIN_TXN: { + unsigned int nOPage = 0; + readUint32(p, &nOPage); + if( pIns==0 ){ + /* Nothing has changed */ + runSql(p, "COMMIT"); + }else if( p->nErr ){ + runSql(p, "ROLLBACK"); + }else{ + int rc; + sqlite3_bind_int64(pIns, 1, nOPage); + sqlite3_bind_null(pIns, 2); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); + p->nPage = nOPage; + runSql(p, "COMMIT"); + } + break; + } + case ORIGIN_PAGE: { + unsigned int pgno = 0; + int rc; + readUint32(p, &pgno); + if( p->nErr ) break; + if( pIns==0 ){ + pIns = prepareStmt(p, + "INSERT INTO sqlite_dbpage(pgno,data) VALUES(?1,?2)" + ); + if( pIns==0 ) break; + } + readBytes(p, szOPage, buf); + if( p->nErr ) break; + p->nPageSent++; + sqlite3_bind_int64(pIns, 1, pgno); + sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed (pgno=%u): %s", + sqlite3_sql(pIns), pgno, sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); + break; + } + default: { + reportError(p, "Unknown message 0x%02x %lld bytes into conversation", + c, p->nIn); + break; + } + } + } + + if( pIns ) sqlite3_finalize(pIns); + closeDb(p); +} + +/* +** The argument might be -vvv...vv with any number of "v"s. Return +** the number of "v"s. Return 0 if the argument is not a -vvv...v. +*/ +static int numVs(const char *z){ + int n = 0; + if( z[0]!='-' ) return 0; + z++; + if( z[0]=='-' ) z++; + while( z[0]=='v' ){ n++; z++; } + if( z[0]==0 ) return n; + return 0; +} + +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static const char *cmdline_option_value(int argc, const char * const*argv, + int i){ + if( i==argc ){ + fprintf(stderr,"%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + +/* +** Return the current time in milliseconds since the Julian epoch. +*/ +sqlite3_int64 currentTime(void){ + sqlite3_int64 now = 0; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + if( pVfs && pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64!=0 ){ + pVfs->xCurrentTimeInt64(pVfs, &now); + } + return now; +} + +/* +** Parse command-line arguments. Dispatch subroutines to do the +** requested work. +** +** Input formats: +** +** (1) sqlite3-rsync FILENAME1 USER@HOST:FILENAME2 +** +** (2) sqlite3-rsync USER@HOST:FILENAME1 FILENAME2 +** +** (3) sqlite3-rsync --origin FILENAME1 +** +** (4) sqlite3-rsync --replica FILENAME2 +** +** The user types (1) or (2). SSH launches (3) or (4). +** +** If (1) is seen then popen2 is used launch (4) on the remote and +** originSide() is called locally. +** +** If (2) is seen, then popen2() is used to launch (3) on the remote +** and replicaSide() is run locally. +** +** If (3) is seen, call originSide() on stdin and stdout. +** +q** If (4) is seen, call replicaSide() on stdin and stdout. +*/ +int main(int argc, char const * const *argv){ + int isOrigin = 0; + int isReplica = 0; + int i; + SQLiteRsync ctx; + char *zDiv; + FILE *pIn = 0; + FILE *pOut = 0; + int childPid = 0; + const char *zSsh = "ssh"; + const char *zExe = "sqlite3-rsync"; + char *zCmd = 0; + sqlite3_int64 tmStart; + sqlite3_int64 tmEnd; + sqlite3_int64 tmElapse; + const char *zRemoteErrFile = 0; + +#define cli_opt_val cmdline_option_value(argc, argv, ++i) + memset(&ctx, 0, sizeof(ctx)); + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( strcmp(z,"--origin")==0 ){ + isOrigin = 1; + continue; + } + if( strcmp(z,"--replica")==0 ){ + isReplica = 1; + continue; + } + if( numVs(z) ){ + ctx.eVerbose += numVs(z); + continue; + } + if( strcmp(z, "--ssh")==0 ){ + zSsh = cli_opt_val; + continue; + } + if( strcmp(z, "--exe")==0 ){ + zExe = cli_opt_val; + continue; + } + if( strcmp(z, "--logfile")==0 ){ + /* DEBUG OPTION: --logfile FILENAME + ** Cause all local output traffic to be duplicated in FILENAME */ + if( ctx.pLog ) fclose(ctx.pLog); + ctx.pLog = fopen(argv[++i],"wb"); + if( ctx.pLog==0 ){ + fprintf(stderr, "cannot open \"%s\" for writing\n", argv[i]); + return 1; + } + continue; + } + if( strcmp(z, "--errorfile")==0 ){ + /* DEBUG OPTION: --errorfile FILENAME + ** Error messages on the local side are written into FILENAME */ + ctx.zErrFile = argv[++i]; + continue; + } + if( strcmp(z, "--remote-errorfile")==0 ){ + /* DEBUG OPTION: --remote-errorfile FILENAME + ** Error messages on the remote side are written into FILENAME on + ** the remote side. */ + zRemoteErrFile = argv[++i]; + continue; + } + if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 + || strcmp(z, "-?")==0 + ){ + printf("%s", zUsage); + return 0; + } + if( strcmp(z, "--version")==0 ){ + printf("%s\n", sqlite3_sourceid()); + return 0; + } + if( z[0]=='-' ){ + if( strcmp(z,"--commcheck")==0 ){ /* DEBUG ONLY */ + /* Run a communication check with the remote side. Do not attempt + ** to exchange any database connection */ + ctx.bCommCheck = 1; + continue; + } + if( strcmp(z,"--arg-escape-check")==0 ){ /* DEBUG ONLY */ + /* Test the append_escaped_arg() routine by using it to render a + ** copy of the input command-line, assuming all arguments except + ** this one are filenames. */ + sqlite3_str *pStr = sqlite3_str_new(0); + int k; + for(k=0; k<argc; k++){ + append_escaped_arg(pStr, argv[k], i!=k); + } + printf("%s\n", sqlite3_str_value(pStr)); + return 0; + } + fprintf(stderr, + "unknown option: \"%s\". Use --help for more detail.\n", z); + return 1; + } + if( ctx.zOrigin==0 ){ + ctx.zOrigin = z; + }else if( ctx.zReplica==0 ){ + ctx.zReplica = z; + }else{ + fprintf(stderr, "Unknown argument: \"%s\"\n", z); + return 1; + } + } + if( ctx.zOrigin==0 ){ + fprintf(stderr, "missing ORIGIN database filename\n"); + return 1; + } + if( ctx.zReplica==0 ){ + fprintf(stderr, "missing REPLICA database filename\n"); + return 1; + } + if( isOrigin && isReplica ){ + fprintf(stderr, "bad option combination\n"); + return 1; + } + if( isOrigin ){ + ctx.pIn = stdin; + ctx.pOut = stdout; + ctx.isRemote = 1; + originSide(&ctx); + return 0; + } + if( isReplica ){ + ctx.pIn = stdin; + ctx.pOut = stdout; + ctx.isRemote = 1; + replicaSide(&ctx); + return 0; + } + if( ctx.zReplica==0 ){ + fprintf(stderr, "missing REPLICA database filename\n"); + return 1; + } + tmStart = currentTime(); + zDiv = strchr(ctx.zOrigin,':'); + if( zDiv ){ + if( strchr(ctx.zReplica,':')!=0 ){ + fprintf(stderr, + "At least one of ORIGIN and REPLICA must be a local database\n" + "You provided two remote databases.\n"); + return 1; + } + /* Remote ORIGIN and local REPLICA */ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + *(zDiv++) = 0; + append_escaped_arg(pStr, ctx.zOrigin, 0); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--origin", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + append_escaped_arg(pStr, zDiv, 1); + append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + replicaSide(&ctx); + }else if( (zDiv = strchr(ctx.zReplica,':'))!=0 ){ + /* Local ORIGIN and remote REPLICA */ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + *(zDiv++) = 0; + append_escaped_arg(pStr, ctx.zReplica, 0); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); + append_escaped_arg(pStr, zDiv, 1); + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + originSide(&ctx); + }else{ + /* Local ORIGIN and REPLICA */ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, argv[0], 1); + append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + append_escaped_arg(pStr, ctx.zOrigin, 1); + append_escaped_arg(pStr, ctx.zReplica, 1); + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + originSide(&ctx); + } + if( ctx.pLog ) fclose(ctx.pLog); + tmEnd = currentTime(); + tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ + if( ctx.nErr ){ + printf("Databases where not synced due to errors\n"); + } + if( ctx.eVerbose>=1 ){ + char *zMsg; + sqlite3_int64 szTotal = (sqlite3_int64)ctx.nPage*(sqlite3_int64)ctx.szPage; + sqlite3_int64 nIO = ctx.nOut +ctx.nIn; + zMsg = sqlite3_mprintf("sent %,lld bytes, received %,lld bytes", + ctx.nOut, ctx.nIn); + printf("%s", zMsg); + sqlite3_free(zMsg); + if( tmElapse>0 ){ + zMsg = sqlite3_mprintf(", %,.2f bytes/sec", + 1000.0*(double)nIO/(double)tmElapse); + printf("%s\n", zMsg); + sqlite3_free(zMsg); + }else{ + printf("\n"); + } + if( ctx.nErr==0 ){ + if( nIO<=szTotal && nIO>0 ){ + zMsg = sqlite3_mprintf("total size %,lld speedup is %.2f", + szTotal, (double)szTotal/(double)nIO); + }else{ + zMsg = sqlite3_mprintf("total size %,lld", szTotal); + } + printf("%s\n", zMsg); + sqlite3_free(zMsg); + } + } + sqlite3_free(zCmd); + if( pIn!=0 && pOut!=0 ){ + pclose2(pIn, pOut, childPid); + } + return ctx.nErr; +} |