diff options
author | danielk1977 <danielk1977@noemail.net> | 2008-03-20 11:04:21 +0000 |
---|---|---|
committer | danielk1977 <danielk1977@noemail.net> | 2008-03-20 11:04:21 +0000 |
commit | f653d78282b61b9ea5940453d0065cbdbdd6ffb2 (patch) | |
tree | 530d6026b9061dde71e524aa694d93850edb1fba /src | |
parent | f219f196f8ece632a514d8f0aa3053c1812e9e23 (diff) | |
download | sqlite-f653d78282b61b9ea5940453d0065cbdbdd6ffb2.tar.gz sqlite-f653d78282b61b9ea5940453d0065cbdbdd6ffb2.zip |
Allow a VACUUM operation to change the page-size in the same way as it can be used to change a database between auto-vacuum and normal mode. (CVS 4896)
FossilOrigin-Name: 871df0e7c36a88f175cfc63797745e52a1b1796b
Diffstat (limited to 'src')
-rw-r--r-- | src/btree.c | 246 | ||||
-rw-r--r-- | src/main.c | 3 | ||||
-rw-r--r-- | src/pager.c | 27 | ||||
-rw-r--r-- | src/pager.h | 5 | ||||
-rw-r--r-- | src/pragma.c | 5 | ||||
-rw-r--r-- | src/sqliteInt.h | 3 | ||||
-rw-r--r-- | src/test2.c | 4 | ||||
-rw-r--r-- | src/vacuum.c | 17 |
8 files changed, 252 insertions, 58 deletions
diff --git a/src/btree.c b/src/btree.c index 82085eb3a..023e5a1aa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.440 2008/03/04 17:45:01 mlcreech Exp $ +** $Id: btree.c,v 1.441 2008/03/20 11:04:21 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -1635,20 +1635,21 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ ** is returned if we run out of memory. */ static int lockBtree(BtShared *pBt){ - int rc, pageSize; + int rc; MemPage *pPage1; assert( sqlite3_mutex_held(pBt->mutex) ); if( pBt->pPage1 ) return SQLITE_OK; rc = sqlite3BtreeGetPage(pBt, 1, &pPage1, 0); if( rc!=SQLITE_OK ) return rc; - /* Do some checking to help insure the file we opened really is ** a valid database file. */ rc = SQLITE_NOTADB; if( sqlite3PagerPagecount(pBt->pPager)>0 ){ + int pageSize; + int usableSize; u8 *page1 = pPage1->aData; if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; @@ -1666,11 +1667,25 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); - pBt->pageSize = pageSize; - pBt->usableSize = pageSize - page1[20]; - if( pBt->usableSize<500 ){ + usableSize = pageSize - page1[20]; + if( pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePage(pPage1); + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; + sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); + return SQLITE_OK; + } + if( usableSize<500 ){ goto page1_init_failed; } + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; pBt->maxEmbedFrac = page1[21]; pBt->minEmbedFrac = page1[22]; pBt->minLeafFrac = page1[23]; @@ -1879,7 +1894,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ #endif do { - if( pBt->pPage1==0 ){ + while( rc==SQLITE_OK && pBt->pPage1==0 ){ rc = lockBtree(pBt); } @@ -2359,7 +2374,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ } } #endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, nTrunc); + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, nTrunc, 0); sqlite3BtreeLeave(p); } return rc; @@ -6729,62 +6744,209 @@ const char *sqlite3BtreeGetJournalname(Btree *p){ ** Copy the complete content of pBtFrom into pBtTo. A transaction ** must be active for both files. ** -** The size of file pBtFrom may be reduced by this operation. -** If anything goes wrong, the transaction on pBtFrom is rolled back. +** The size of file pTo may be reduced by this operation. +** If anything goes wrong, the transaction on pTo is rolled back. +** +** If successful, CommitPhaseOne() may be called on pTo before returning. +** The caller should finish committing the transaction on pTo by calling +** sqlite3BtreeCommit(). */ static int btreeCopyFile(Btree *pTo, Btree *pFrom){ int rc = SQLITE_OK; - Pgno i, nPage, nToPage, iSkip; + Pgno i; + + Pgno nFromPage; /* Number of pages in pFrom */ + Pgno nToPage; /* Number of pages in pTo */ + Pgno nNewPage; /* Number of pages in pTo after the copy */ + + Pgno iSkip; /* Pending byte page in pTo */ + int nToPageSize; /* Page size of pTo in bytes */ + int nFromPageSize; /* Page size of pFrom in bytes */ BtShared *pBtTo = pTo->pBt; BtShared *pBtFrom = pFrom->pBt; pBtTo->db = pTo->db; pBtFrom->db = pFrom->db; - + + nToPageSize = pBtTo->pageSize; + nFromPageSize = pBtFrom->pageSize; if( pTo->inTrans!=TRANS_WRITE || pFrom->inTrans!=TRANS_WRITE ){ return SQLITE_ERROR; } - if( pBtTo->pCursor ) return SQLITE_BUSY; + if( pBtTo->pCursor ){ + return SQLITE_BUSY; + } + nToPage = sqlite3PagerPagecount(pBtTo->pPager); - nPage = sqlite3PagerPagecount(pBtFrom->pPager); + nFromPage = sqlite3PagerPagecount(pBtFrom->pPager); iSkip = PENDING_BYTE_PAGE(pBtTo); - for(i=1; rc==SQLITE_OK && i<=nPage; i++){ - DbPage *pDbPage; - if( i==iSkip ) continue; - rc = sqlite3PagerGet(pBtFrom->pPager, i, &pDbPage); - if( rc ) break; - rc = sqlite3PagerOverwrite(pBtTo->pPager, i, sqlite3PagerGetData(pDbPage)); - sqlite3PagerUnref(pDbPage); - } - /* If the file is shrinking, journal the pages that are being truncated - ** so that they can be rolled back if the commit fails. + /* Variable nNewPage is the number of pages required to store the + ** contents of pFrom using the current page-size of pTo. */ - for(i=nPage+1; rc==SQLITE_OK && i<=nToPage; i++){ - DbPage *pDbPage; - if( i==iSkip ) continue; - rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); - if( rc ) break; - rc = sqlite3PagerWrite(pDbPage); - sqlite3PagerDontWrite(pDbPage); - /* Yeah. It seems wierd to call DontWrite() right after Write(). But - ** that is because the names of those procedures do not exactly - ** represent what they do. Write() really means "put this page in the - ** rollback journal and mark it as dirty so that it will be written - ** to the database file later." DontWrite() undoes the second part of - ** that and prevents the page from being written to the database. The - ** page is still on the rollback journal, though. And that is the whole - ** point of this loop: to put pages on the rollback journal. */ - sqlite3PagerUnref(pDbPage); + nNewPage = ((i64)nFromPage * (i64)nFromPageSize + (i64)nToPageSize - 1) / + (i64)nToPageSize; + + for(i=1; rc==SQLITE_OK && (i<=nToPage || i<=nNewPage); i++){ + + /* Journal the original page. + ** + ** iSkip is the page number of the locking page (PENDING_BYTE_PAGE) + ** in database *pTo (before the copy). This page is never written + ** into the journal file. Unless i==iSkip or the page was not + ** present in pTo before the copy operation, journal page i from pTo. + */ + if( i!=iSkip && i<=nToPage ){ + DbPage *pDbPage; + rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); + if( rc ){ + break; + } + rc = sqlite3PagerWrite(pDbPage); + if( rc ){ + break; + } + if( i>nFromPage ){ + /* Yeah. It seems wierd to call DontWrite() right after Write(). But + ** that is because the names of those procedures do not exactly + ** represent what they do. Write() really means "put this page in the + ** rollback journal and mark it as dirty so that it will be written + ** to the database file later." DontWrite() undoes the second part of + ** that and prevents the page from being written to the database. The + ** page is still on the rollback journal, though. And that is the + ** whole point of this block: to put pages on the rollback journal. + */ + sqlite3PagerDontWrite(pDbPage); + } + sqlite3PagerUnref(pDbPage); + } + + /* Overwrite the data in page i of the target database */ + if( rc==SQLITE_OK && i!=iSkip && i<=nNewPage ){ + + DbPage *pToPage = 0; + sqlite3_int64 iOff; + + rc = sqlite3PagerGet(pBtTo->pPager, i, &pToPage); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pToPage); + } + + for( + iOff=(i-1)*nToPageSize; + rc==SQLITE_OK && iOff<i*nToPageSize; + iOff += nFromPageSize + ){ + DbPage *pFromPage = 0; + Pgno iFrom = (iOff/nFromPageSize)+1; + + if( iFrom==PENDING_BYTE_PAGE(pBtFrom) ){ + continue; + } + + rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage); + if( rc==SQLITE_OK ){ + char *zTo = sqlite3PagerGetData(pToPage); + char *zFrom = sqlite3PagerGetData(pFromPage); + int nCopy; + + if( nFromPageSize>=nToPageSize ){ + zFrom += ((i-1)*nToPageSize - ((iFrom-1)*nFromPageSize)); + nCopy = nToPageSize; + }else{ + zTo += (((iFrom-1)*nFromPageSize) - (i-1)*nToPageSize); + nCopy = nFromPageSize; + } + + memcpy(zTo, zFrom, nCopy); + sqlite3PagerUnref(pFromPage); + } + } + + if( pToPage ) sqlite3PagerUnref(pToPage); + } } - if( !rc && nPage<nToPage ){ - rc = sqlite3PagerTruncate(pBtTo->pPager, nPage); + + /* If things have worked so far, the database file may need to be + ** truncated. The complex part is that it may need to be truncated to + ** a size that is not an integer multiple of nToPageSize - the current + ** page size used by the pager associated with B-Tree pTo. + ** + ** For example, say the page-size of pTo is 2048 bytes and the original + ** number of pages is 5 (10 KB file). If pFrom has a page size of 1024 + ** bytes and 9 pages, then the file needs to be truncated to 9KB. + */ + if( rc==SQLITE_OK ){ + if( nFromPageSize!=nToPageSize ){ + sqlite3_file *pFile = sqlite3PagerFile(pBtTo->pPager); + i64 iSize = (i64)nFromPageSize * (i64)nFromPage; + i64 iNow = (i64)((nToPage>nNewPage)?nToPage:nNewPage) * (i64)nToPageSize; + i64 iPending = ((i64)PENDING_BYTE_PAGE(pBtTo)-1) *(i64)nToPageSize; + + assert( iSize<=iNow ); + + /* Commit phase one syncs the journal file associated with pTo + ** containing the original data. It does not sync the database file + ** itself. After doing this it is safe to use OsTruncate() and other + ** file APIs on the database file directly. + */ + pBtTo->db = pTo->db; + rc = sqlite3PagerCommitPhaseOne(pBtTo->pPager, 0, 0, 1); + if( iSize<iNow && rc==SQLITE_OK ){ + rc = sqlite3OsTruncate(pFile, iSize); + } + + /* The loop that copied data from database pFrom to pTo did not + ** populate the locking page of database pTo. If the page-size of + ** pFrom is smaller than that of pTo, this means some data will + ** not have been copied. + ** + ** This block copies the missing data from database pFrom to pTo + ** using file APIs. This is safe because at this point we know that + ** all of the original data from pTo has been synced into the + ** journal file. At this point it would be safe to do anything at + ** all to the database file except truncate it to zero bytes. + */ + if( rc==SQLITE_OK && nFromPageSize<nToPageSize && iSize>iPending){ + i64 iOff; + for( + iOff=iPending; + rc==SQLITE_OK && iOff<(iPending+nToPageSize); + iOff += nFromPageSize + ){ + DbPage *pFromPage = 0; + Pgno iFrom = (iOff/nFromPageSize)+1; + + if( iFrom==PENDING_BYTE_PAGE(pBtFrom) || iFrom>nFromPage ){ + continue; + } + + rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage); + if( rc==SQLITE_OK ){ + char *zFrom = sqlite3PagerGetData(pFromPage); + rc = sqlite3OsWrite(pFile, zFrom, nFromPageSize, iOff); + sqlite3PagerUnref(pFromPage); + } + } + } + + /* Sync the database file */ + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSync(pBtTo->pPager); + } + }else{ + rc = sqlite3PagerTruncate(pBtTo->pPager, nNewPage); + } + if( rc==SQLITE_OK ){ + pBtTo->pageSizeFixed = 0; + } } if( rc ){ sqlite3BtreeRollback(pTo); } + return rc; } int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ diff --git a/src/main.c b/src/main.c index dd9999e53..4fef038b2 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,7 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.424 2008/03/19 16:08:54 drh Exp $ +** $Id: main.c,v 1.425 2008/03/20 11:04:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include <ctype.h> @@ -971,6 +971,7 @@ static int openDatabase( db->aDb = db->aDbStatic; db->autoCommit = 1; db->nextAutovac = -1; + db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames #if SQLITE_DEFAULT_FILE_FORMAT<4 | SQLITE_LegacyFileFmt diff --git a/src/pager.c b/src/pager.c index 5313404cc..7f76c13d5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.419 2008/03/20 04:45:49 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.420 2008/03/20 11:04:21 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -4502,6 +4502,17 @@ static int pager_incr_changecounter(Pager *pPager, int isDirect){ } /* +** Sync the pager file to disk. +*/ +int sqlite3PagerSync(Pager *pPager){ + int rc; + pagerEnter(pPager); + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + pagerLeave(pPager); + return rc; +} + +/* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual ** journal file. zMaster may be NULL, which is interpreted as no master @@ -4517,8 +4528,18 @@ static int pager_incr_changecounter(Pager *pPager, int isDirect){ ** ** If parameter nTrunc is non-zero, then the pager file is truncated to ** nTrunc pages (this is used by auto-vacuum databases). +** +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. */ -int sqlite3PagerCommitPhaseOne(Pager *pPager, const char *zMaster, Pgno nTrunc){ +int sqlite3PagerCommitPhaseOne( + Pager *pPager, + const char *zMaster, + Pgno nTrunc, + int noSync +){ int rc = SQLITE_OK; PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", @@ -4630,7 +4651,7 @@ int sqlite3PagerCommitPhaseOne(Pager *pPager, const char *zMaster, Pgno nTrunc){ pPager->pDirty = 0; /* Sync the database file. */ - if( !pPager->noSync ){ + if( !pPager->noSync && !noSync ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } IOTRACE(("DBSYNC %p\n", pPager)) diff --git a/src/pager.h b/src/pager.h index c864a1501..a6bd54c97 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.69 2008/02/02 20:47:38 drh Exp $ +** @(#) $Id: pager.h,v 1.70 2008/03/20 11:04:21 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -73,7 +73,7 @@ int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void*); int sqlite3PagerPagecount(Pager*); int sqlite3PagerTruncate(Pager*,Pgno); int sqlite3PagerBegin(DbPage*, int exFlag); -int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, Pgno); +int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, Pgno, int); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); int sqlite3PagerIsreadonly(Pager*); @@ -95,6 +95,7 @@ void *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); int sqlite3PagerLockingMode(Pager *, int); void *sqlite3PagerTempSpace(Pager*); +int sqlite3PagerSync(Pager *pPager); #if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) && !defined(SQLITE_OMIT_DISKIO) int sqlite3PagerReleaseMemory(int); diff --git a/src/pragma.c b/src/pragma.c index 067426d11..98d7d2a0f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code used to implement the PRAGMA command. ** -** $Id: pragma.c,v 1.171 2008/03/19 00:21:31 drh Exp $ +** $Id: pragma.c,v 1.172 2008/03/20 11:04:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include <ctype.h> @@ -350,7 +350,8 @@ void sqlite3Pragma( /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, atoi(zRight), -1) ){ + db->nextPagesize = atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1) ){ db->mallocFailed = 1; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b27971ae1..fdf4d7879 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.675 2008/03/20 00:32:20 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.676 2008/03/20 11:04:21 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -575,6 +575,7 @@ struct sqlite3 { u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ + int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ i64 lastRowid; /* ROWID of most recent insert (see above) */ diff --git a/src/test2.c b/src/test2.c index d326a91d8..a8457efb3 100644 --- a/src/test2.c +++ b/src/test2.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test2.c,v 1.55 2008/03/19 00:21:31 drh Exp $ +** $Id: test2.c,v 1.56 2008/03/20 11:04:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -164,7 +164,7 @@ static int pager_commit( return TCL_ERROR; } pPager = sqlite3TextToPtr(argv[1]); - rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); + rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, errorName(rc), 0); return TCL_ERROR; diff --git a/src/vacuum.c b/src/vacuum.c index c6544ac57..2bfd7aea9 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -14,7 +14,7 @@ ** Most of the code in this file may be omitted by defining the ** SQLITE_OMIT_VACUUM macro. ** -** $Id: vacuum.c,v 1.76 2008/01/03 00:01:25 drh Exp $ +** $Id: vacuum.c,v 1.77 2008/03/20 11:04:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -85,6 +85,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ char *zSql = 0; /* SQL statements */ int saved_flags; /* Saved value of the db->flags */ Db *pDb = 0; /* Database to detach at end of vacuum */ + int nRes; /* Save the current value of the write-schema flag before setting it. */ saved_flags = db->flags; @@ -112,13 +113,15 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ pDb = &db->aDb[db->nDb-1]; assert( strcmp(db->aDb[db->nDb-1].zName,"vacuum_db")==0 ); pTemp = db->aDb[db->nDb-1].pBt; - sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), - sqlite3BtreeGetReserve(pMain)); - if( db->mallocFailed ){ + + nRes = sqlite3BtreeGetReserve(pMain); + if( sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), nRes) + || sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes) + || db->mallocFailed + ){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - assert( sqlite3BtreeGetPageSize(pTemp)==sqlite3BtreeGetPageSize(pMain) ); rc = execSql(db, "PRAGMA vacuum_db.synchronous=OFF"); if( rc!=SQLITE_OK ){ goto end_of_vacuum; @@ -237,6 +240,10 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = sqlite3BtreeCommit(pMain); } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes); + } + end_of_vacuum: /* Restore the original value of db->flags */ db->flags = saved_flags; |