diff options
| -rw-r--r-- | src/backend/storage/file/fd.c | 186 |
1 files changed, 126 insertions, 60 deletions
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 78c7d41ac48..436b901c20d 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -43,8 +43,8 @@ * wrappers around fopen(3), opendir(3), popen(3) and open(2), respectively. * They behave like the corresponding native functions, except that the handle * is registered with the current subtransaction, and will be automatically - * closed at abort. These are intended for short operations like reading a - * configuration file, and there is a fixed limit on the number of files that + * closed at abort. These are intended mainly for short operations like + * reading a configuration file; there is a limit on the number of files that * can be opened using these functions at any one time. * * Finally, BasicOpenFile is just a thin wrapper around open() that can @@ -198,13 +198,7 @@ static uint64 temporary_files_size = 0; /* * List of OS handles opened with AllocateFile, AllocateDir and * OpenTransientFile. - * - * Since we don't want to encourage heavy use of those functions, - * it seems OK to put a pretty small maximum limit on the number of - * simultaneously allocated descs. */ -#define MAX_ALLOCATED_DESCS 32 - typedef enum { AllocateDescFile, @@ -216,17 +210,18 @@ typedef enum typedef struct { AllocateDescKind kind; + SubTransactionId create_subid; union { FILE *file; DIR *dir; int fd; } desc; - SubTransactionId create_subid; } AllocateDesc; static int numAllocatedDescs = 0; -static AllocateDesc allocatedDescs[MAX_ALLOCATED_DESCS]; +static int maxAllocatedDescs = 0; +static AllocateDesc *allocatedDescs = NULL; /* * Number of temporary files opened during the current session; @@ -252,6 +247,7 @@ static int nextTempTableSpace = 0; * Insert - put a file at the front of the Lru ring * LruInsert - put a file at the front of the Lru ring and open it * ReleaseLruFile - Release an fd by closing the last entry in the Lru ring + * ReleaseLruFiles - Release fd(s) until we're under the max_safe_fds limit * AllocateVfd - grab a free (or new) file record (from VfdArray) * FreeVfd - free a file record * @@ -279,11 +275,14 @@ static void LruDelete(File file); static void Insert(File file); static int LruInsert(File file); static bool ReleaseLruFile(void); +static void ReleaseLruFiles(void); static File AllocateVfd(void); static void FreeVfd(File file); static int FileAccess(File file); static File OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError); +static bool reserveAllocatedDesc(void); +static int FreeDesc(AllocateDesc *desc); static void AtProcExit_Files(int code, Datum arg); static void CleanupTempFiles(bool isProcExit); static void RemovePgTempFilesInDir(const char *tmpdirname); @@ -693,11 +692,8 @@ LruInsert(File file) if (FileIsNotOpen(file)) { - while (nfile + numAllocatedDescs >= max_safe_fds) - { - if (!ReleaseLruFile()) - break; - } + /* Close excess kernel FDs. */ + ReleaseLruFiles(); /* * The open could still fail for lack of file descriptors, eg due to @@ -736,6 +732,9 @@ LruInsert(File file) return 0; } +/* + * Release one kernel FD by closing the least-recently-used VFD. + */ static bool ReleaseLruFile(void) { @@ -754,6 +753,20 @@ ReleaseLruFile(void) return false; /* no files available to free */ } +/* + * Release kernel FDs as needed to get under the max_safe_fds limit. + * After calling this, it's OK to try to open another file. + */ +static void +ReleaseLruFiles(void) +{ + while (nfile + numAllocatedDescs >= max_safe_fds) + { + if (!ReleaseLruFile()) + break; + } +} + static File AllocateVfd(void) { @@ -907,11 +920,8 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode) file = AllocateVfd(); vfdP = &VfdCache[file]; - while (nfile + numAllocatedDescs >= max_safe_fds) - { - if (!ReleaseLruFile()) - break; - } + /* Close excess kernel FDs. */ + ReleaseLruFiles(); vfdP->fd = BasicOpenFile(fileName, fileFlags, fileMode); @@ -1491,6 +1501,66 @@ FilePathName(File file) /* + * Make room for another allocatedDescs[] array entry if needed and possible. + * Returns true if an array element is available. + */ +static bool +reserveAllocatedDesc(void) +{ + AllocateDesc *newDescs; + int newMax; + + /* Quick out if array already has a free slot. */ + if (numAllocatedDescs < maxAllocatedDescs) + return true; + + /* + * If the array hasn't yet been created in the current process, initialize + * it with FD_MINFREE / 2 elements. In many scenarios this is as many as + * we will ever need, anyway. We don't want to look at max_safe_fds + * immediately because set_max_safe_fds() may not have run yet. + */ + if (allocatedDescs == NULL) + { + newMax = FD_MINFREE / 2; + newDescs = (AllocateDesc *) malloc(newMax * sizeof(AllocateDesc)); + /* Out of memory already? Treat as fatal error. */ + if (newDescs == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* + * Consider enlarging the array beyond the initial allocation used above. + * By the time this happens, max_safe_fds should be known accurately. + * + * We mustn't let allocated descriptors hog all the available FDs, and in + * practice we'd better leave a reasonable number of FDs for VFD use. So + * set the maximum to max_safe_fds / 2. (This should certainly be at + * least as large as the initial size, FD_MINFREE / 2.) + */ + newMax = max_safe_fds / 2; + if (newMax > maxAllocatedDescs) + { + newDescs = (AllocateDesc *) realloc(allocatedDescs, + newMax * sizeof(AllocateDesc)); + /* Treat out-of-memory as a non-fatal error. */ + if (newDescs == NULL) + return false; + allocatedDescs = newDescs; + maxAllocatedDescs = newMax; + return true; + } + + /* Can't enlarge allocatedDescs[] any more. */ + return false; +} + +/* * Routines that want to use stdio (ie, FILE*) should use AllocateFile * rather than plain fopen(). This lets fd.c deal with freeing FDs if * necessary to open the file. When done, call FreeFile rather than fclose. @@ -1515,16 +1585,15 @@ AllocateFile(const char *name, const char *mode) DO_DB(elog(LOG, "AllocateFile: Allocated %d (%s)", numAllocatedDescs, name)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedFiles[]; the test against max_safe_fds prevents AllocateFile - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open file \"%s\"", - name); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"", + maxAllocatedDescs, name))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); TryAgain: if ((file = fopen(name, mode)) != NULL) @@ -1566,16 +1635,15 @@ OpenTransientFile(FileName fileName, int fileFlags, int fileMode) DO_DB(elog(LOG, "OpenTransientFile: Allocated %d (%s)", numAllocatedDescs, fileName)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedFiles[]; the test against max_safe_fds prevents BasicOpenFile - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open file \"%s\"", - fileName); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to open file \"%s\"", + maxAllocatedDescs, fileName))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); fd = BasicOpenFile(fileName, fileFlags, fileMode); @@ -1607,16 +1675,15 @@ OpenPipeStream(const char *command, const char *mode) DO_DB(elog(LOG, "OpenPipeStream: Allocated %d (%s)", numAllocatedDescs, command)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedFiles[]; the test against max_safe_fds prevents AllocateFile - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to execute command \"%s\"", - command); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to execute command \"%s\"", + maxAllocatedDescs, command))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); TryAgain: fflush(stdout); @@ -1759,16 +1826,15 @@ AllocateDir(const char *dirname) DO_DB(elog(LOG, "AllocateDir: Allocated %d (%s)", numAllocatedDescs, dirname)); - /* - * The test against MAX_ALLOCATED_DESCS prevents us from overflowing - * allocatedDescs[]; the test against max_safe_fds prevents AllocateDir - * from hogging every one of the available FDs, which'd lead to infinite - * looping. - */ - if (numAllocatedDescs >= MAX_ALLOCATED_DESCS || - numAllocatedDescs >= max_safe_fds - 1) - elog(ERROR, "exceeded MAX_ALLOCATED_DESCS while trying to open directory \"%s\"", - dirname); + /* Can we allocate another non-virtual FD? */ + if (!reserveAllocatedDesc()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("exceeded maxAllocatedDescs (%d) while trying to open directory \"%s\"", + maxAllocatedDescs, dirname))); + + /* Close excess kernel FDs. */ + ReleaseLruFiles(); TryAgain: if ((dir = opendir(dirname)) != NULL) |