diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/backend/replication/Makefile | 1 | ||||
| -rw-r--r-- | src/backend/replication/basebackup.c | 81 | ||||
| -rw-r--r-- | src/backend/replication/basebackup_copy.c | 21 | ||||
| -rw-r--r-- | src/backend/replication/basebackup_server.c | 302 | ||||
| -rw-r--r-- | src/backend/utils/activity/wait_event.c | 6 | ||||
| -rw-r--r-- | src/bin/pg_basebackup/pg_basebackup.c | 208 | ||||
| -rw-r--r-- | src/bin/pg_basebackup/t/010_pg_basebackup.pl | 64 | ||||
| -rw-r--r-- | src/include/replication/basebackup_sink.h | 3 | ||||
| -rw-r--r-- | src/include/utils/wait_event.h | 2 |
9 files changed, 626 insertions, 62 deletions
diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile index 74b97cf126a..a8f4757f0c0 100644 --- a/src/backend/replication/Makefile +++ b/src/backend/replication/Makefile @@ -19,6 +19,7 @@ OBJS = \ basebackup.o \ basebackup_copy.o \ basebackup_progress.o \ + basebackup_server.o \ basebackup_sink.o \ basebackup_throttle.o \ repl_gram.o \ diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 3afbbe7e02e..d32da515355 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -55,8 +55,10 @@ typedef enum { + BACKUP_TARGET_BLACKHOLE, BACKUP_TARGET_COMPAT, - BACKUP_TARGET_CLIENT + BACKUP_TARGET_CLIENT, + BACKUP_TARGET_SERVER } backup_target_type; typedef struct @@ -69,6 +71,7 @@ typedef struct uint32 maxrate; bool sendtblspcmapfile; backup_target_type target; + char *target_detail; backup_manifest_option manifest; pg_checksum_type manifest_checksum_type; } basebackup_options; @@ -702,6 +705,8 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_manifest = false; bool o_manifest_checksums = false; bool o_target = false; + bool o_target_detail = false; + char *target_str = "compat"; /* placate compiler */ MemSet(opt, 0, sizeof(*opt)); opt->target = BACKUP_TARGET_COMPAT; @@ -847,25 +852,35 @@ parse_basebackup_options(List *options, basebackup_options *opt) } else if (strcmp(defel->defname, "target") == 0) { - char *optval = defGetString(defel); + target_str = defGetString(defel); if (o_target) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); - if (strcmp(optval, "client") == 0) + if (strcmp(target_str, "blackhole") == 0) + opt->target = BACKUP_TARGET_BLACKHOLE; + else if (strcmp(target_str, "client") == 0) opt->target = BACKUP_TARGET_CLIENT; + else if (strcmp(target_str, "server") == 0) + opt->target = BACKUP_TARGET_SERVER; else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized target: \"%s\"", optval))); + errmsg("unrecognized target: \"%s\"", target_str))); o_target = true; } - else - ereport(ERROR, - errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", - defel->defname)); + else if (strcmp(defel->defname, "target_detail") == 0) + { + char *optval = defGetString(defel); + + if (o_target_detail) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("duplicate option \"%s\"", defel->defname))); + opt->target_detail = optval; + o_target_detail = true; + } } if (opt->label == NULL) opt->label = "base backup"; @@ -877,6 +892,22 @@ parse_basebackup_options(List *options, basebackup_options *opt) errmsg("manifest checksums require a backup manifest"))); opt->manifest_checksum_type = CHECKSUM_TYPE_NONE; } + if (opt->target == BACKUP_TARGET_SERVER) + { + if (opt->target_detail == NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("target '%s' requires a target detail", + target_str))); + } + else + { + if (opt->target_detail != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("target '%s' does not accept a target detail", + target_str))); + } } @@ -908,14 +939,38 @@ SendBaseBackup(BaseBackupCmd *cmd) /* * If the TARGET option was specified, then we can use the new copy-stream - * protocol. If not, we must fall back to the old and less capable - * copy-tablespace protocol. + * protocol. If the target is specifically 'client' then set up to stream + * the backup to the client; otherwise, it's being sent someplace else and + * should not be sent to the client. + * + * If the TARGET option was not specified, we must fall back to the older + * and less capable copy-tablespace protocol. */ - if (opt.target != BACKUP_TARGET_COMPAT) - sink = bbsink_copystream_new(); + if (opt.target == BACKUP_TARGET_CLIENT) + sink = bbsink_copystream_new(true); + else if (opt.target != BACKUP_TARGET_COMPAT) + sink = bbsink_copystream_new(false); else sink = bbsink_copytblspc_new(); + /* + * If a non-default backup target is in use, arrange to send the data + * wherever it needs to go. + */ + switch (opt.target) + { + case BACKUP_TARGET_BLACKHOLE: + /* Nothing to do, just discard data. */ + break; + case BACKUP_TARGET_COMPAT: + case BACKUP_TARGET_CLIENT: + /* Nothing to do, handling above is sufficient. */ + break; + case BACKUP_TARGET_SERVER: + sink = bbsink_server_new(sink, opt.target_detail); + break; + } + /* Set up network throttling, if client requested it */ if (opt.maxrate > 0) sink = bbsink_throttle_new(sink, opt.maxrate); diff --git a/src/backend/replication/basebackup_copy.c b/src/backend/replication/basebackup_copy.c index 8dfdff06449..938e19a6a43 100644 --- a/src/backend/replication/basebackup_copy.c +++ b/src/backend/replication/basebackup_copy.c @@ -44,6 +44,9 @@ typedef struct bbsink_copystream /* Common information for all types of sink. */ bbsink base; + /* Are we sending the archives to the client, or somewhere else? */ + bool send_to_client; + /* * Protocol message buffer. We assemble CopyData protocol messages by * setting the first character of this buffer to 'd' (archive or manifest @@ -131,11 +134,12 @@ const bbsink_ops bbsink_copytblspc_ops = { * Create a new 'copystream' bbsink. */ bbsink * -bbsink_copystream_new(void) +bbsink_copystream_new(bool send_to_client) { bbsink_copystream *sink = palloc0(sizeof(bbsink_copystream)); *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_copystream_ops; + sink->send_to_client = send_to_client; /* Set up for periodic progress reporting. */ sink->last_progress_report_time = GetCurrentTimestamp(); @@ -212,8 +216,12 @@ bbsink_copystream_archive_contents(bbsink *sink, size_t len) StringInfoData buf; uint64 targetbytes; - /* Send the archive content to the client (with leading type byte). */ - pq_putmessage('d', mysink->msgbuffer, len + 1); + /* Send the archive content to the client, if appropriate. */ + if (mysink->send_to_client) + { + /* Add one because we're also sending a leading type byte. */ + pq_putmessage('d', mysink->msgbuffer, len + 1); + } /* Consider whether to send a progress report to the client. */ targetbytes = mysink->bytes_done_at_last_time_check @@ -294,8 +302,11 @@ bbsink_copystream_manifest_contents(bbsink *sink, size_t len) { bbsink_copystream *mysink = (bbsink_copystream *) sink; - /* Send the manifest content to the client (with leading type byte). */ - pq_putmessage('d', mysink->msgbuffer, len + 1); + if (mysink->send_to_client) + { + /* Add one because we're also sending a leading type byte. */ + pq_putmessage('d', mysink->msgbuffer, len + 1); + } } /* diff --git a/src/backend/replication/basebackup_server.c b/src/backend/replication/basebackup_server.c new file mode 100644 index 00000000000..ce1b7b47977 --- /dev/null +++ b/src/backend/replication/basebackup_server.c @@ -0,0 +1,302 @@ +/*------------------------------------------------------------------------- + * + * basebackup_server.c + * store basebackup archives on the server + * + * IDENTIFICATION + * src/backend/replication/basebackup_server.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "replication/basebackup.h" +#include "replication/basebackup_sink.h" +#include "storage/fd.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +typedef struct bbsink_server +{ + /* Common information for all types of sink. */ + bbsink base; + + /* Directory in which backup is to be stored. */ + char *pathname; + + /* Currently open file (or 0 if nothing open). */ + File file; + + /* Current file position. */ + off_t filepos; +} bbsink_server; + +static void bbsink_server_begin_archive(bbsink *sink, + const char *archive_name); +static void bbsink_server_archive_contents(bbsink *sink, size_t len); +static void bbsink_server_end_archive(bbsink *sink); +static void bbsink_server_begin_manifest(bbsink *sink); +static void bbsink_server_manifest_contents(bbsink *sink, size_t len); +static void bbsink_server_end_manifest(bbsink *sink); + +const bbsink_ops bbsink_server_ops = { + .begin_backup = bbsink_forward_begin_backup, + .begin_archive = bbsink_server_begin_archive, + .archive_contents = bbsink_server_archive_contents, + .end_archive = bbsink_server_end_archive, + .begin_manifest = bbsink_server_begin_manifest, + .manifest_contents = bbsink_server_manifest_contents, + .end_manifest = bbsink_server_end_manifest, + .end_backup = bbsink_forward_end_backup, + .cleanup = bbsink_forward_cleanup +}; + +/* + * Create a new 'server' bbsink. + */ +bbsink * +bbsink_server_new(bbsink *next, char *pathname) +{ + bbsink_server *sink = palloc0(sizeof(bbsink_server)); + + *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_server_ops; + sink->pathname = pathname; + sink->base.bbs_next = next; + + /* Replication permission is not sufficient in this case. */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create server backup"))); + + /* + * It's not a good idea to store your backups in the same directory that + * you're backing up. If we allowed a relative path here, that could easily + * happen accidentally, so we don't. The user could still accomplish the + * same thing by including the absolute path to $PGDATA in the pathname, + * but that's likely an intentional bad decision rather than an accident. + */ + if (!is_absolute_path(pathname)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_NAME), + errmsg("relative path not allowed for server backup"))); + + switch (pg_check_dir(pathname)) + { + case 0: + /* + * Does not exist, so create it using the same permissions we'd use + * for a new subdirectory of the data directory itself. + */ + if (MakePGDirectory(pathname) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", pathname))); + break; + + case 1: + /* Exists, empty. */ + break; + + case 2: + case 3: + case 4: + /* Exists, not empty. */ + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_FILE), + errmsg("directory \"%s\" exists but is not empty", + pathname))); + break; + + default: + /* Access problem. */ + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not access directory \"%s\": %m", + pathname))); + } + + return &sink->base; +} + +/* + * Open the correct output file for this archive. + */ +static void +bbsink_server_begin_archive(bbsink *sink, const char *archive_name) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *filename; + + Assert(mysink->file == 0); + Assert(mysink->filepos == 0); + + filename = psprintf("%s/%s", mysink->pathname, archive_name); + + mysink->file = PathNameOpenFile(filename, + O_CREAT | O_EXCL | O_WRONLY | PG_BINARY); + if (mysink->file <= 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filename))); + + pfree(filename); + + bbsink_forward_begin_archive(sink, archive_name); +} + +/* + * Write the data to the output file. + */ +static void +bbsink_server_archive_contents(bbsink *sink, size_t len) +{ + bbsink_server *mysink = (bbsink_server *) sink; + int nbytes; + + nbytes = FileWrite(mysink->file, mysink->base.bbs_buffer, len, + mysink->filepos, WAIT_EVENT_BASEBACKUP_WRITE); + + if (nbytes != len) + { + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + FilePathName(mysink->file)), + errhint("Check free disk space."))); + /* short write: complain appropriately */ + ereport(ERROR, + (errcode(ERRCODE_DISK_FULL), + errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u", + FilePathName(mysink->file), + nbytes, (int) len, (unsigned) mysink->filepos), + errhint("Check free disk space."))); + } + + mysink->filepos += nbytes; + + bbsink_forward_archive_contents(sink, len); +} + +/* + * fsync and close the current output file. + */ +static void +bbsink_server_end_archive(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + + /* + * We intentionally don't use data_sync_elevel here, because the server + * shouldn't PANIC just because we can't guarantee the the backup has been + * written down to disk. Running recovery won't fix anything in this case + * anyway. + */ + if (FileSync(mysink->file, WAIT_EVENT_BASEBACKUP_SYNC) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + FilePathName(mysink->file)))); + + + /* We're done with this file now. */ + FileClose(mysink->file); + mysink->file = 0; + mysink->filepos = 0; + + bbsink_forward_end_archive(sink); +} + +/* + * Open the output file to which we will write the manifest. + * + * Just like pg_basebackup, we write the manifest first under a temporary + * name and then rename it into place after fsync. That way, if the manifest + * is there and under the correct name, the user can be sure that the backup + * completed. + */ +static void +bbsink_server_begin_manifest(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *tmp_filename; + + Assert(mysink->file == 0); + + tmp_filename = psprintf("%s/backup_manifest.tmp", mysink->pathname); + + mysink->file = PathNameOpenFile(tmp_filename, + O_CREAT | O_EXCL | O_WRONLY | PG_BINARY); + if (mysink->file <= 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", tmp_filename))); + + pfree(tmp_filename); + + bbsink_forward_begin_manifest(sink); +} + +/* + * Each chunk of manifest data is sent using a CopyData message. + */ +static void +bbsink_server_manifest_contents(bbsink *sink, size_t len) +{ + bbsink_server *mysink = (bbsink_server *) sink; + int nbytes; + + nbytes = FileWrite(mysink->file, mysink->base.bbs_buffer, len, + mysink->filepos, WAIT_EVENT_BASEBACKUP_WRITE); + + if (nbytes != len) + { + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + FilePathName(mysink->file)), + errhint("Check free disk space."))); + /* short write: complain appropriately */ + ereport(ERROR, + (errcode(ERRCODE_DISK_FULL), + errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u", + FilePathName(mysink->file), + nbytes, (int) len, (unsigned) mysink->filepos), + errhint("Check free disk space."))); + } + + mysink->filepos += nbytes; + + bbsink_forward_manifest_contents(sink, len); +} + +/* + * fsync the backup manifest, close the file, and then rename it into place. + */ +static void +bbsink_server_end_manifest(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *tmp_filename; + char *filename; + + /* We're done with this file now. */ + FileClose(mysink->file); + mysink->file = 0; + + /* + * Rename it into place. This also fsyncs the temporary file, so we don't + * need to do that here. We don't use data_sync_elevel here for the same + * reasons as in bbsink_server_end_archive. + */ + tmp_filename = psprintf("%s/backup_manifest.tmp", mysink->pathname); + filename = psprintf("%s/backup_manifest", mysink->pathname); + durable_rename(tmp_filename, filename, ERROR); + pfree(filename); + pfree(tmp_filename); + + bbsink_forward_end_manifest(sink); +} diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 0f5f18f02e0..021b83de7a3 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -522,6 +522,12 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_BASEBACKUP_READ: event_name = "BaseBackupRead"; break; + case WAIT_EVENT_BASEBACKUP_SYNC: + event_name = "BaseBackupSync"; + break; + case WAIT_EVENT_BASEBACKUP_WRITE: + event_name = "BaseBackupWrite"; + break; case WAIT_EVENT_BUFFILE_READ: event_name = "BufFileRead"; break; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 2a58be638a5..ec3b4f3c174 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -115,7 +115,7 @@ typedef enum static char *basedir = NULL; static TablespaceList tablespace_dirs = {NULL, NULL}; static char *xlog_dir = NULL; -static char format = 'p'; /* p(lain)/t(ar) */ +static char format = '\0'; /* p(lain)/t(ar) */ static char *label = "pg_basebackup base backup"; static bool noclean = false; static bool checksum_failure = false; @@ -132,6 +132,7 @@ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ static char *replication_slot = NULL; static bool temp_replication_slot = true; +static char *backup_target = NULL; static bool create_slot = false; static bool no_slot = false; static bool verify_checksums = true; @@ -364,6 +365,8 @@ usage(void) printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nOptions controlling the output:\n")); + printf(_(" -t, --target=TARGET[:DETAIL]\n" + " backup target (if other than client)\n")); printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n")); printf(_(" -F, --format=p|t output format (plain (default), tar)\n")); printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n" @@ -1232,15 +1235,22 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) } /* - * Create an appropriate backup streamer. We know that - * recovery GUCs are supported, because this protocol can only - * be used on v15+. + * Create an appropriate backup streamer, unless a backup + * target was specified. In that case, it's up to the server + * to put the backup wherever it needs to go. */ - state->streamer = - CreateBackupStreamer(archive_name, - spclocation, - &state->manifest_inject_streamer, - true, false); + if (backup_target == NULL) + { + /* + * We know that recovery GUCs are supported, because this + * protocol can only be used on v15+. + */ + state->streamer = + CreateBackupStreamer(archive_name, + spclocation, + &state->manifest_inject_streamer, + true, false); + } break; } @@ -1312,24 +1322,32 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) GetCopyDataEnd(r, copybuf, cursor); /* - * If we're supposed inject the manifest into the archive, we - * prepare to buffer it in memory; otherwise, we prepare to - * write it to a temporary file. + * If a backup target was specified, figuring out where to put + * the manifest is the server's problem. Otherwise, we need to + * deal with it. */ - if (state->manifest_inject_streamer != NULL) - state->manifest_buffer = createPQExpBuffer(); - else + if (backup_target == NULL) { - snprintf(state->manifest_filename, - sizeof(state->manifest_filename), - "%s/backup_manifest.tmp", basedir); - state->manifest_file = - fopen(state->manifest_filename, "wb"); - if (state->manifest_file == NULL) + /* + * If we're supposed inject the manifest into the archive, + * we prepare to buffer it in memory; otherwise, we + * prepare to write it to a temporary file. + */ + if (state->manifest_inject_streamer != NULL) + state->manifest_buffer = createPQExpBuffer(); + else { - pg_log_error("could not create file \"%s\": %m", - state->manifest_filename); - exit(1); + snprintf(state->manifest_filename, + sizeof(state->manifest_filename), + "%s/backup_manifest.tmp", basedir); + state->manifest_file = + fopen(state->manifest_filename, "wb"); + if (state->manifest_file == NULL) + { + pg_log_error("could not create file \"%s\": %m", + state->manifest_filename); + exit(1); + } } } break; @@ -1698,13 +1716,41 @@ BaseBackup(void) if (manifest) { AppendStringCommandOption(&buf, use_new_option_syntax, "MANIFEST", - manifest_force_encode ? "force-encode" : "yes"); + manifest_force_encode ? "force-encode" : "yes"); if (manifest_checksums != NULL) AppendStringCommandOption(&buf, use_new_option_syntax, - "MANIFEST_CHECKSUMS", manifest_checksums); + "MANIFEST_CHECKSUMS", manifest_checksums); } - if (serverMajor >= 1500) + if (backup_target != NULL) + { + char *colon; + + if (serverMajor < 1500) + { + pg_log_error("backup targets are not supported by this server version"); + exit(1); + } + + AppendPlainCommandOption(&buf, use_new_option_syntax, "TABLESPACE_MAP"); + + if ((colon = strchr(backup_target, ':')) == NULL) + { + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET", backup_target); + } + else + { + char *target; + + target = pnstrdup(backup_target, colon - backup_target); + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET", target); + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET_DETAIL", colon + 1); + } + } + else if (serverMajor >= 1500) AppendStringCommandOption(&buf, use_new_option_syntax, "TARGET", "client"); @@ -1799,8 +1845,13 @@ BaseBackup(void) * Verify tablespace directories are empty. Don't bother with the * first once since it can be relocated, and it will be checked before * we do anything anyway. + * + * Note that this is skipped for tar format backups and backups that + * the server is storing to a target location, since in that case + * we won't be storing anything into these directories and thus should + * not create them. */ - if (format == 'p' && !PQgetisnull(res, i, 1)) + if (backup_target == NULL && format == 'p' && !PQgetisnull(res, i, 1)) { char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1))); @@ -1811,7 +1862,8 @@ BaseBackup(void) /* * When writing to stdout, require a single tablespace */ - writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0; + writing_to_stdout = format == 't' && basedir != NULL && + strcmp(basedir, "-") == 0; if (writing_to_stdout && PQntuples(res) > 1) { pg_log_error("can only write single tablespace to stdout, database has %d", @@ -1894,7 +1946,7 @@ BaseBackup(void) res = PQgetResult(conn); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - pg_log_error("could not get write-ahead log end position from server: %s", + pg_log_error("backup failed: %s", PQerrorMessage(conn)); exit(1); } @@ -2028,8 +2080,11 @@ BaseBackup(void) * synced after being completed. In plain format, all the data of the * base directory is synced, taking into account all the tablespaces. * Errors are not considered fatal. + * + * If, however, there's a backup target, we're not writing anything + * locally, so in that case we skip this step. */ - if (do_sync) + if (do_sync && backup_target == NULL) { if (verbose) pg_log_info("syncing data to disk ..."); @@ -2051,7 +2106,7 @@ BaseBackup(void) * without a backup_manifest file, decreasing the chances that a directory * we leave behind will be mistaken for a valid backup. */ - if (!writing_to_stdout && manifest) + if (!writing_to_stdout && manifest && backup_target == NULL) { char tmp_filename[MAXPGPATH]; char filename[MAXPGPATH]; @@ -2085,6 +2140,7 @@ main(int argc, char **argv) {"max-rate", required_argument, NULL, 'r'}, {"write-recovery-conf", no_argument, NULL, 'R'}, {"slot", required_argument, NULL, 'S'}, + {"target", required_argument, NULL, 't'}, {"tablespace-mapping", required_argument, NULL, 'T'}, {"wal-method", required_argument, NULL, 'X'}, {"gzip", no_argument, NULL, 'z'}, @@ -2135,7 +2191,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", + while ((c = getopt_long(argc, argv, "CD:F:r:RS:t:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", long_options, &option_index)) != -1) { switch (c) @@ -2176,6 +2232,9 @@ main(int argc, char **argv) case 2: no_slot = true; break; + case 't': + backup_target = pg_strdup(optarg); + break; case 'T': tablespace_list_append(optarg); break; @@ -2308,27 +2367,72 @@ main(int argc, char **argv) } /* - * Required arguments + * Setting the backup target to 'client' is equivalent to leaving out the + * option. This logic allows us to assume elsewhere that the backup is + * being stored locally if and only if backup_target == NULL. + */ + if (backup_target != NULL && strcmp(backup_target, "client") == 0) + { + pg_free(backup_target); + backup_target = NULL; + } + + /* + * Can't use --format with --target. Without --target, default format is + * tar. + */ + if (backup_target != NULL && format != '\0') + { + pg_log_error("cannot specify both format and backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + if (format == '\0') + format = 'p'; + + /* + * Either directory or backup target should be specified, but not both */ - if (basedir == NULL) + if (basedir == NULL && backup_target == NULL) { - pg_log_error("no target directory specified"); + pg_log_error("must specify output directory or backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + if (basedir != NULL && backup_target != NULL) + { + pg_log_error("cannot specify both output directory and backup target"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } /* - * Mutually exclusive arguments + * Compression doesn't make sense unless tar format is in use. */ if (format == 'p' && compresslevel != 0) { - pg_log_error("only tar mode backups can be compressed"); + if (backup_target == NULL) + pg_log_error("only tar mode backups can be compressed"); + else + pg_log_error("client-side compression is not possible when a backup target is specfied"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } + /* + * Sanity checks for WAL method. + */ + if (backup_target != NULL && includewal == STREAM_WAL) + { + pg_log_error("WAL cannot be streamed when a backup target is specified"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0) { pg_log_error("cannot stream write-ahead logs in tar mode to stdout"); @@ -2345,6 +2449,9 @@ main(int argc, char **argv) exit(1); } + /* + * Sanity checks for replication slot options. + */ if (no_slot) { if (replication_slot) @@ -2378,8 +2485,18 @@ main(int argc, char **argv) } } + /* + * Sanity checks on WAL directory. + */ if (xlog_dir) { + if (backup_target != NULL) + { + pg_log_error("WAL directory location cannot be specified along with a backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } if (format != 'p') { pg_log_error("WAL directory location can only be specified in plain mode"); @@ -2400,6 +2517,7 @@ main(int argc, char **argv) } #ifndef HAVE_LIBZ + /* Sanity checks for compression level. */ if (compresslevel != 0) { pg_log_error("this build does not support compression"); @@ -2407,6 +2525,9 @@ main(int argc, char **argv) } #endif + /* + * Sanity checks for progress reporting options. + */ if (showprogress && !estimatesize) { pg_log_error("%s and %s are incompatible options", @@ -2416,6 +2537,9 @@ main(int argc, char **argv) exit(1); } + /* + * Sanity checks for backup manifest options. + */ if (!manifest && manifest_checksums != NULL) { pg_log_error("%s and %s are incompatible options", @@ -2458,11 +2582,11 @@ main(int argc, char **argv) manifest = false; /* - * Verify that the target directory exists, or create it. For plaintext - * backups, always require the directory. For tar backups, require it - * unless we are writing to stdout. + * If an output directory was specified, verify that it exists, or create + * it. Note that for a tar backup, an output directory of "-" means we are + * writing to stdout, so do nothing in that case. */ - if (format == 'p' || strcmp(basedir, "-") != 0) + if (basedir != NULL && (format == 'p' || strcmp(basedir, "-") != 0)) verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata); /* determine remote server's xlog segment size */ diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index f0243f28d42..f7e21941ebf 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -10,7 +10,7 @@ use File::Path qw(rmtree); use Fcntl qw(:seek); use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; -use Test::More tests => 115; +use Test::More tests => 135; program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -474,6 +474,68 @@ $node->command_ok( ], 'pg_basebackup -X stream runs with --no-slot'); rmtree("$tempdir/backupnoslot"); +$node->command_ok( + [ @pg_basebackup_defs, '-D', "$tempdir/backupxf", '-X', 'fetch' ], + 'pg_basebackup -X fetch runs'); + +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'blackhole' ], + qr/WAL cannot be streamed when a backup target is specified/, + 'backup target requires -X'); +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'blackhole', '-X', 'stream' ], + qr/WAL cannot be streamed when a backup target is specified/, + 'backup target requires -X other than -X stream'); +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'bogus', '-X', 'none' ], + qr/unrecognized target/, + 'backup target unrecognized'); +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'blackhole', '-X', 'none', '-D', "$tempdir/blackhole" ], + qr/cannot specify both output directory and backup target/, + 'backup target and output directory'); +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'blackhole', '-X', 'none', '-Ft' ], + qr/cannot specify both format and backup target/, + 'backup target and output directory'); +$node->command_ok( + [ @pg_basebackup_defs, '--target', 'blackhole', '-X', 'none' ], + 'backup target blackhole'); +$node->command_ok( + [ @pg_basebackup_defs, '--target', "server:$tempdir/backuponserver", '-X', 'none' ], + 'backup target server'); +ok(-f "$tempdir/backuponserver/base.tar", 'backup tar was created'); +rmtree("$tempdir/backuponserver"); + +$node->command_fails( + [ + @pg_basebackup_defs, '-D', + "$tempdir/backupxs_sl_fail", '-X', + 'stream', '-S', + 'slot0' + ], + 'pg_basebackup fails with nonexistent replication slot'); + +$node->command_fails( + [ @pg_basebackup_defs, '-D', "$tempdir/backupxs_slot", '-C' ], + 'pg_basebackup -C fails without slot name'); + +$node->command_fails( + [ + @pg_basebackup_defs, '-D', + "$tempdir/backupxs_slot", '-C', + '-S', 'slot0', + '--no-slot' + ], + 'pg_basebackup fails with -C -S --no-slot'); +$node->command_fails_like( + [ @pg_basebackup_defs, '--target', 'blackhole', '-D', "$tempdir/blackhole" ], + qr/cannot specify both output directory and backup target/, + 'backup target and output directory'); + +$node->command_ok( + [ @pg_basebackup_defs, '-D', "$tempdir/backuptr/co", '-X', 'none' ], + 'pg_basebackup -X fetch runs'); $node->command_fails( [ diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h index 25436defa8f..4acadf406dd 100644 --- a/src/include/replication/basebackup_sink.h +++ b/src/include/replication/basebackup_sink.h @@ -282,9 +282,10 @@ extern void bbsink_forward_end_backup(bbsink *sink, XLogRecPtr endptr, extern void bbsink_forward_cleanup(bbsink *sink); /* Constructors for various types of sinks. */ -extern bbsink *bbsink_copystream_new(void); +extern bbsink *bbsink_copystream_new(bool send_to_client); extern bbsink *bbsink_copytblspc_new(void); extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size); +extern bbsink *bbsink_server_new(bbsink *next, char *pathname); extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate); /* Extra interface functions for progress reporting. */ diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index e0b2f56f47e..395d325c5fe 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -157,6 +157,8 @@ typedef enum typedef enum { WAIT_EVENT_BASEBACKUP_READ = PG_WAIT_IO, + WAIT_EVENT_BASEBACKUP_SYNC, + WAIT_EVENT_BASEBACKUP_WRITE, WAIT_EVENT_BUFFILE_READ, WAIT_EVENT_BUFFILE_WRITE, WAIT_EVENT_BUFFILE_TRUNCATE, |