Skip to content

Repair backup failed (the value of segment_size is greater than 2G) #510

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Repair backup failed (the value of segment_size is greater than 2G) #510

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1b9311e
Repair backup failed (the value of segment_size is greater than 2G)
Jul 14, 2022
45c7297
Repair backup failed (the value of segment_size is greater than 2G)
Jul 14, 2022
b41e02c
fix Memory leak
Oct 17, 2022
97927d9
Merge branch 'master' into merge510
hslightdb Oct 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/catalog.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1117,7 +1117,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
file->segno = (int) segno;

if (get_control_value_int64(buf, "n_blocks", &n_blocks, false))
file->n_blocks = (int) n_blocks;
file->n_blocks = (int64) n_blocks;

if (get_control_value_int64(buf, "n_headers", &n_headers, false))
file->n_headers = (int) n_headers;
Expand Down Expand Up @@ -2568,7 +2568,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);

if (file->n_blocks > 0)
len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
len += sprintf(line+len, ",\"n_blocks\":\"%ld\"", file->n_blocks);

if (file->n_headers > 0)
{
Expand Down
28 changes: 14 additions & 14 deletions src/data.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn,
while (!page_is_valid && try_again--)
{
/* read the block */
int read_len = fio_pread(in, page, blknum * BLCKSZ);
int read_len = fio_pread(in, page, ((int64)blknum) * BLCKSZ);

/* The block could have been truncated. It is fine. */
if (read_len == 0)
Expand Down Expand Up @@ -609,7 +609,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
}

file->read_size = rc * BLCKSZ;
file->read_size = ((int64)rc) * BLCKSZ;

/* refresh n_blocks for FULL and DELTA */
if (backup_mode == BACKUP_MODE_FULL ||
Expand Down Expand Up @@ -758,7 +758,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
}

file->read_size = rc * BLCKSZ;
file->read_size = ((int64)rc) * BLCKSZ;

/* Determine that file didn`t changed in case of incremental catchup */
if (backup_mode != BACKUP_MODE_FULL &&
Expand Down Expand Up @@ -1072,7 +1072,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
if (fio_fseek(out, 0) < 0)
elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno));

if (fio_ftruncate(out, blknum * BLCKSZ) != 0)
if (fio_ftruncate(out, ((int64)blknum) * BLCKSZ) != 0)
elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno));

break;
Expand Down Expand Up @@ -1123,7 +1123,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
cur_pos_in != headers[n_hdr].pos)
{
if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0)
elog(ERROR, "Cannot seek to offset %u of \"%s\": %s",
elog(ERROR, "Cannot seek to offset %ld of \"%s\": %s",
headers[n_hdr].pos, from_fullpath, strerror(errno));

cur_pos_in = headers[n_hdr].pos;
Expand Down Expand Up @@ -1158,7 +1158,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
* When restoring file from FULL backup, pages are written sequentially,
* so there is no need to issue fseek for every page.
*/
write_pos = blknum * BLCKSZ;
write_pos = ((int64)blknum) * BLCKSZ;

if (cur_pos_out != write_pos)
{
Expand Down Expand Up @@ -1742,7 +1742,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
elog(ERROR, "Cannot seek block %u of \"%s\": %s",
blknum, fullpath, strerror(errno));
else
elog(INFO, "Seek to %u", headers[n_hdr].pos);
elog(INFO, "Seek to %ld", headers[n_hdr].pos);

cur_pos_in = headers[n_hdr].pos;
}
Expand Down Expand Up @@ -1879,7 +1879,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
/* read local data file and construct map with block checksums */
PageState*
get_checksum_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)
int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)
{
PageState *checksum_map = NULL;
FILE *in = NULL;
Expand All @@ -1894,7 +1894,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version,

/* truncate up to blocks */
if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0)
elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s",
elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s",
n_blocks, fullpath, strerror(errno));

setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE);
Expand Down Expand Up @@ -1948,7 +1948,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version,
/* return bitmap of valid blocks, bitmap is empty, then NULL is returned */
datapagemap_t *
get_lsn_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno)
int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno)
{
FILE *in = NULL;
BlockNumber blknum = 0;
Expand All @@ -1965,7 +1965,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version,

/* truncate up to blocks */
if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0)
elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s",
elog(ERROR, "Cannot truncate file to blknum %ld \"%s\": %s",
n_blocks, fullpath, strerror(errno));

setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE);
Expand Down Expand Up @@ -2296,9 +2296,9 @@ copy_pages(const char *to_fullpath, const char *from_fullpath,

else if (rc == PageIsOk)
{
if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0)
elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s",
blknum * BLCKSZ, to_fullpath, strerror(errno));
if (fseek(out, ((int64)blknum) * BLCKSZ, SEEK_SET) != 0)
elog(ERROR, "Cannot seek to position %ld in destination file \"%s\": %s",
((int64)blknum) * BLCKSZ, to_fullpath, strerror(errno));

if (write_page(file, out, curr_page) != BLCKSZ)
elog(ERROR, "File: \"%s\", cannot write at block %u: %s",
Expand Down
10 changes: 5 additions & 5 deletions src/pg_probackup.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ typedef struct pgFile
Oid relOid; /* relOid extracted from path, if applicable */
ForkName forkName; /* forkName extracted from path, if applicable */
int segno; /* Segment number for ptrack */
int n_blocks; /* number of blocks in the data file in data directory */
int64 n_blocks; /* number of blocks in the data file in data directory */
bool is_cfs; /* Flag to distinguish files compressed by CFS*/
int external_dir_num; /* Number of external directory. 0 if not external */
bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */
Expand Down Expand Up @@ -677,8 +677,8 @@ typedef struct BackupPageHeader
typedef struct BackupPageHeader2
{
XLogRecPtr lsn;
int32 block; /* block number */
int32 pos; /* position in backup file */
int64 block; /* block number */
int64 pos; /* position in backup file */
uint16 checksum;
} BackupPageHeader2;

Expand Down Expand Up @@ -1115,9 +1115,9 @@ extern bool create_empty_file(fio_location from_location, const char *to_root,
fio_location to_location, pgFile *file);

extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
int64 n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno);
int64 n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno);
extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map);

Expand Down
4 changes: 2 additions & 2 deletions src/utils/file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2138,7 +2138,7 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file,

COMP_FILE_CRC32(true, file->crc, buf, hdr.size);

if (fio_fseek(out, blknum * BLCKSZ) < 0)
if (fio_fseek(out, ((int64)blknum) * BLCKSZ) < 0)
{
elog(ERROR, "Cannot seek block %u of \"%s\": %s",
blknum, to_fullpath, strerror(errno));
Expand Down Expand Up @@ -2196,7 +2196,7 @@ fio_send_pages_impl(int out, char* buf)
datapagemap_iterator_t *iter = NULL;
/* page headers */
int32 hdr_num = -1;
int32 cur_pos_out = 0;
int64 cur_pos_out = 0;
BackupPageHeader2 *headers = NULL;

/* open source file */
Expand Down