diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/backup.c | 2 | ||||
| -rw-r--r-- | src/os.c | 15 | ||||
| -rw-r--r-- | src/os.h | 3 | ||||
| -rw-r--r-- | src/os_unix.c | 213 | ||||
| -rw-r--r-- | src/pager.c | 221 | ||||
| -rw-r--r-- | src/pager.h | 3 | ||||
| -rw-r--r-- | src/sqlite.h.in | 30 | ||||
| -rw-r--r-- | src/wal.c | 22 | ||||
| -rw-r--r-- | src/wal.h | 3 |
9 files changed, 208 insertions, 304 deletions
diff --git a/src/backup.c b/src/backup.c index 8939bc67b..71a8a1a3e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); + rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } @@ -140,15 +140,12 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap( - sqlite3_file *id, /* Database file handle */ - int flags, /* SQLITE_MREMAP_XXX flags */ - i64 iOff, /* Offset at which mapping(s) start */ - i64 nOld, /* Size of old mapping */ - i64 nNew, /* Size of requested mapping */ - void **pp /* IN/OUT: Pointer to mapped region */ -){ - return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); + +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, void *p){ + return id->pMethods->xUnfetch(id, p); } /* @@ -259,7 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 2f03c3967..334719200 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,9 +249,14 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ - void *pMapRegion; /* Area memory mapped */ + #endif + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ + int nFetchOut; /* Number of outstanding xFetch refs */ + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -1805,6 +1810,9 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1817,6 +1825,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3074,7 +3083,6 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3085,6 +3093,21 @@ static int unixRead( ); #endif + /* Deal with as much of this write request as possible by transfering + ** data to the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3157,7 +3180,6 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3190,6 +3212,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offset<pFile->mmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3470,6 +3507,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } +#endif /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will @@ -3478,7 +3516,6 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nByte<pFile->mmapSize ){ pFile->mmapSize = nByte; } -#endif return SQLITE_OK; } @@ -3568,6 +3605,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3635,8 +3685,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_GETFD: { - *(int*)pArg = pFile->h; + case SQLITE_FCNTL_MMAP_SIZE: { + pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } #ifdef SQLITE_DEBUG @@ -4451,91 +4501,86 @@ static int unixShmUnmap( */ #define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) -/* -** Map, remap or unmap part of the database file. -*/ -static int unixMremap( - sqlite3_file *fd, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - unixFile *p = (unixFile *)fd; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ - - assert( iOff==0 ); - /* assert( p->mmapSize==nOld ); */ - assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); - - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + munmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} + +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(p->h, &statbuf); - if( rc==SQLITE_OK && nNew>statbuf.st_size ){ - rc = robust_ftruncate(p->h, nNew); + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; } - /* According to some sources, the effect of changing the size of the - ** underlying file on mapped regions that correspond to the added or - ** removed pages is undefined. However, there is reason to believe that - ** on modern platforms like Linux or OSX, things just work. For example, - ** it is possible to create a mapping larger than the file on disk and - ** extend the file on disk later on. - ** - ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() - ** calls required if the file size is changing. In this case all mappings - ** are rounded up to the nearest 4MB. And if a new mapping is requested - ** that has the same rounded size as an old mapping, the old mapping can - ** be reused as is. */ -#if defined(__APPLE__) || defined(__linux__) - nNewRnd = ROUNDUP(nNew, 4096*1024); - nOldRnd = ROUNDUP(nOld, 4096*1024); -#else - nNewRnd = ROUNDUP(nNew, 4096*1); - nOldRnd = ROUNDUP(nOld, 4096*1); -#endif + if( nMap!=pFd->mmapSize ){ + void *pNew; + unixUnmapfile(pFd); - /* On OSX or Linux, reuse the old mapping if it is the right size. */ -#if defined(__APPLE__) || defined(__linux__) - if( nNewRnd==nOldRnd ){ - VVA_ONLY( p->mmapSize = nNew; ) - return SQLITE_OK; + if( nMap>0 ){ + void *pNew; + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MREMAP; + } + + pFd->pMapRegion = pNew; + pFd->mmapOrigsize = pFd->mmapSize = nMap; + } } -#endif - /* If we get this far, unmap any old mapping. */ - if( nOldRnd!=0 ){ - void *pOld = *ppMap; - munmap(pOld, nOldRnd); - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); - } - - /* And, if required, use mmap() to create a new mapping. */ - if( nNewRnd>0 ){ - int flags = PROT_READ; - if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); - if( pNew==MAP_FAILED ){ - pNew = 0; - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) - rc = SQLITE_IOERR_MREMAP; - }else{ - VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) + return SQLITE_OK; +} + +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +static int unixUnfetch(sqlite3_file *fd, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + assert( (p==0)==(pFd->nFetchOut==0) ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); } - *ppMap = pNew; - return rc; + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -4597,7 +4642,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ - unixMremap, /* xMremap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4865,7 +4911,6 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index 09cc4a237..7e00e1431 100644 --- a/src/pager.c +++ b/src/pager.c @@ -656,14 +656,10 @@ struct Pager { int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - void *pMap; /* Memory mapped prefix of database file */ - i64 nMap; /* Size of mapping at pMap in bytes */ - i64 nMapValid; /* Bytes at pMap known to be valid */ - i64 nMapLimit; /* Maximum permitted mapping size */ + u8 bUseFetch; /* True to use xFetch() */ int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ - int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2088,24 +2084,6 @@ static void pagerReportSize(Pager *pPager){ #endif /* -** Write nBuf bytes of data from buffer pBuf to offset iOff of the -** database file. If this part of the database file is memory mapped, -** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ - int rc = SQLITE_OK; - if( pPager->nMapValid>=(iOff+nBuf) ){ - memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); - }else{ - rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); - } - return rc; -} - -/* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. ** The page begins at offset *pOffset into the file. The *pOffset @@ -2279,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2534,9 +2512,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - if( newSize<pPager->nMapValid ){ - pPager->nMapValid = newSize; - } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -2884,13 +2859,9 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ - memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); - }else{ - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; } } @@ -3120,6 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); } return rc; @@ -3382,22 +3354,24 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Set Pager.nMapLimit, the maximum permitted mapping size, based on the -** current values of Pager.nMapCfgLimit and Pager.pageSize. -** -** If this connection should not use mmap at all, set nMapLimit to zero. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. */ static void pagerFixMaplimit(Pager *pPager){ - if( isOpen(pPager->fd)==0 - || pPager->fd->pMethods->iVersion<3 - || pPager->fd->pMethods->xMremap==0 - || pPager->tempFile - ){ - pPager->nMapLimit = 0; - }else if( pPager->nMapCfgLimit<0 ){ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + if( pPager->bUseFetch ){ + void *p; + i64 nMapLimit; + if( pPager->nMapCfgLimit<0 ){ + nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } + + p = (void *)&nMapLimit; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + } } } @@ -3871,66 +3845,21 @@ static int pagerSyncHotJournal(Pager *pPager){ } /* -** Unmap any memory mapping of the database file. -*/ -static int pagerUnmap(Pager *pPager){ - assert( pPager->nMmapOut==0 ); - if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); - pPager->nMap = 0; - pPager->nMapValid = 0; - } - return SQLITE_OK; -} - -/* -** Create, or recreate, the memory mapping of the database file. -*/ -static int pagerMap(Pager *pPager, int bExtend){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPg; /* Size of mapping to request in pages */ - i64 sz; /* Size of mapping to request in bytes */ - - assert( isOpen(pPager->fd) && pPager->tempFile==0 ); - assert( pPager->pMap==0 || pPager->nMap>0 ); - /* assert( pPager->eState>=1 ); */ - assert( pPager->nMmapOut==0 ); - assert( pPager->nMapLimit>0 ); - - /* Figure out how large a mapping to request. Set variable sz to this - ** value in bytes. */ - nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; - sz = (i64)nPg * pPager->pageSize; - if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - - if( sz!=pPager->nMapValid ){ - int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); - rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); - if( rc==SQLITE_OK ){ - assert( pPager->pMap!=0 ); - pPager->nMap = sz; - }else{ - assert( pPager->pMap==0 ); - pPager->nMap = 0; - } - pPager->nMapValid = pPager->nMap; - } - pPager->bMapResize = 0; - - return rc; -} - -/* ** Obtain a reference to a memory mapped page object for page number pgno. -** The caller must ensure that page pgno lies within the currently mapped -** region. If successful, set *ppPage to point to the new page reference +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set ** *ppPage to zero. ** ** Page references obtained by calling this function should be released ** by calling pagerReleaseMapPage(). */ -static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ PgHdr *p; /* Memory mapped page to return */ if( pPager->pFree ){ @@ -3955,8 +3884,8 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ assert( p->pPager==pPager ); assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; p->pgno = pgno; + p->pData = pData; pPager->nMmapOut++; return SQLITE_OK; @@ -3971,6 +3900,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->nMmapOut--; pPg->pDirty = pPager->pFree; pPager->pFree = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, pPg->pData); } /* @@ -4006,7 +3938,6 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pagerUnmap(pPager); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; @@ -4217,46 +4148,6 @@ static int syncJournal(Pager *pPager, int newHdr){ } /* -** This is called by the wal.c module at the start of a checkpoint. If the -** checkpoint runs to completion, it will set the database file size to -** szReq bytes. This function performs two tasks: -** -** * If the file is currently less than szReq bytes in size, an -** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS -** layer of the expected file size, and -** -** * If mmap is being used, then the mapping is extended to szReq -** bytes in size. -** -** SQLITE_OK is returned if successful, or an error code if an error occurs. -*/ -int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ - int rc; - i64 sz; /* Size of file on disk in bytes */ - - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->nMmapOut==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - if( rc==SQLITE_OK ){ - if( sz>szReq ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); - } - } - - - if( rc==SQLITE_OK ){ - i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; - if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); - } - } - - return rc; -} - -/* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the ** in-memory pages in the list to the database file. The argument may @@ -4315,11 +4206,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; - - if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ - pPager->dbFileSize = pPager->dbSize; - rc = pagerMap(pPager, 1); - } } while( rc==SQLITE_OK && pList ){ @@ -4344,7 +4230,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5164,7 +5050,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->pMap + || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5188,13 +5074,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 || pPager->pMap ){ + if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - if( pPager->pMap ){ - memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); - }else{ - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - } + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ goto failed; } @@ -5211,9 +5093,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - pagerUnmap(pPager); - }else if( pPager->pMap ){ - pPager->bMapResize = 1; + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0); + } } } @@ -5325,7 +5207,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 + const int bMmapOk = (pgno!=1 && pPager->bUseFetch && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5349,15 +5231,20 @@ int sqlite3PagerAcquire( } if( iFrame==0 && bMmapOk ){ - if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager, 0); - } - if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, pData); } if( pPg ){ assert( rc==SQLITE_OK ); @@ -7117,7 +7004,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); @@ -7155,8 +7042,6 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - pagerUnmap(pPager); - if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; diff --git a/src/pager.h b/src/pager.h index 970b5035b..066d1f11d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -176,9 +176,6 @@ int sqlite3SectorSize(sqlite3_file *); void sqlite3PagerTruncateImage(Pager*,Pgno); int sqlite3PagerSetFilesize(Pager *, i64); -/* Write data to the database file */ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); - #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c0a280378..666a2e084 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -707,24 +707,6 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. -** -** Assuming parameter nNew is non-zero, the xMremap method should attempt -** to memory map a region nNew bytes in size starting at offset iOffset -** of the file. If successful, it should set *ppMap to point to the -** mapping and return SQLITE_OK. If the file is opened for read-write -** access, then the mapping should also be read-write. -** -** If nOld is non-zero, then the initial value of *ppMap points to a -** mapping returned by a previous call to xMremap. The existing mapping -** is nOld bytes in size and starts at offset iOffset of the file. In -** this case the xMremap method is expected to unmap the existing mapping -** and overwrite *ppMap with the pointer to the new mapping. If nOld is -** zero, then the initial value of *ppMap is undefined. -** -** If nNew is zero, then no new mapping should be created. Any old -** mapping must still be unmapped if nOld is non-zero. If the nOld -** parameter is non-zero, then the existing mapping is always unmapped - -** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -747,14 +729,12 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, int flags, - sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; -#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ - /* ** CAPI3REF: Standard File Control Opcodes ** @@ -906,6 +886,10 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +** <li>[[SQLITE_FCNTL_MMAP_SIZE]] +** The argument is assumed to point to a value of type sqlite3_int64. An +** advisory maximum amount of this file to memory map in bytes. +** ** </ul> */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -924,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_GETFD 17 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -412,7 +412,6 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ - Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1252,7 +1251,6 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1293,7 +1291,6 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; - pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1725,19 +1722,18 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file is currently smaller than mxPage pages in size, - ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to - ** inform it that it is likely to grow to that size. - ** - ** Additionally, if the pager is using mmap(), then the call to - ** SetFilesize() guarantees that the mapping is not larger than mxPage - ** pages. This makes the sqlite3OsTruncate() call below safe - no pages - ** that are part of the mapped region will be truncated away. */ + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSize<nReq ){ + sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -1749,7 +1745,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -53,8 +53,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen( - sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ |