[PBCKP-314] rewrite fio_mkdir to use pio

Makefile

@@ -95,7 +95,7 @@ borrowed.mk: $(firstword $(MAKEFILE_LIST))
 	$(file >$@,# This file is autogenerated. Do not edit!)
 	$(foreach borrowed_file, $(BORROWED_H_SRC) $(BORROWED_C_SRC), \
 		$(file >>$@,$(addprefix $(BORROW_DIR)/, $(notdir $(borrowed_file))): | $(CURDIR)/$(BORROW_DIR)/ $(realpath $(top_srcdir)/$(borrowed_file))) \
-		$(file >>$@,$(shell echo "	"'$$(LN_S) $(realpath $(top_srcdir)/$(borrowed_file)) $$@')) \
+		$(file >>$@,$(shell echo "	"'$$(LN_S) -f $(realpath $(top_srcdir)/$(borrowed_file)) $$@')) \
 	)
 include borrowed.mk
 

src/backup.c

@@ -111,6 +111,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 	time_t		start_time, end_time;
 	char		pretty_time[20];
 	char		pretty_bytes[20];
+	err_i		err = $noerr();
+
 
 	elog(INFO, "Database backup start");
 	if(current.external_dir_str)
@@ -252,7 +254,12 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 		char stream_xlog_path[MAXPGPATH];
 
 		join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR);
-		fio_mkdir(FIO_BACKUP_HOST, stream_xlog_path, DIR_PERMISSION, false);
+		err = $i(pioMakeDir, current.backup_location, .path = stream_xlog_path,
+				 .mode = DIR_PERMISSION, .strict = false);
+		if ($haserr(err))
+		{
+			elog(ERROR, "Can not create WAL directory: %s", $errmsg(err));
+		}
 
 		start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt,
 							current.start_lsn, current.tli, true);
@@ -400,7 +407,13 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
 				join_path_components(dirpath, current.database_dir, file->rel_path);
 
 			elog(LOG, "Create directory '%s'", dirpath);
-			fio_mkdir(FIO_BACKUP_HOST, dirpath, DIR_PERMISSION, false);
+			err = $i(pioMakeDir, current.backup_location, .path = dirpath,
+					 .mode = DIR_PERMISSION, .strict = false);
+			if ($haserr(err))
+			{
+				elog(ERROR, "Can not create instance backup directory: %s",
+					 $errmsg(err));
+			}
 		}
 
 	}

src/catalog.c

@@ -22,7 +22,7 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo);
 static pgBackup* get_oldest_backup(timelineInfo *tlinfo);
 static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
 static pgBackup *readBackupControlFile(const char *path);
-static int create_backup_dir(pgBackup *backup, const char *backup_instance_path);
+static err_i create_backup_dir(pgBackup *backup, const char *backup_instance_path);
 
 static bool backup_lock_exit_hook_registered = false;
 static parray *locks = NULL;
@@ -1461,9 +1461,11 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path)
 	int	i;
 	char	temp[MAXPGPATH];
 	parray *subdirs;
+	err_i	err = $noerr();
 
 	/* Try to create backup directory at first */
-	if (create_backup_dir(backup, backup_instance_path) != 0)
+	err = create_backup_dir(backup, backup_instance_path);
+	if ($haserr(err))
 	{
 		/* Clear backup_id as indication of error */
 		backup->backup_id = INVALID_BACKUP_ID;
@@ -1499,7 +1501,12 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path)
 	for (i = 0; i < parray_num(subdirs); i++)
 	{
 		join_path_components(temp, backup->root_dir, parray_get(subdirs, i));
-		fio_mkdir(FIO_BACKUP_HOST, temp, DIR_PERMISSION, false);
+		err = $i(pioMakeDir, backup->backup_location, .path = temp,
+				 .mode = DIR_PERMISSION, .strict = false);
+		if ($haserr(err))
+		{
+			elog(ERROR, "Can not create backup directory: %s", $errmsg(err));
+		}
 	}
 
 	free_dir_list(subdirs);
@@ -1512,22 +1519,25 @@ pgBackupInitDir(pgBackup *backup, const char *backup_instance_path)
  *  0 - ok
  * -1 - error (warning message already emitted)
  */
-int
+static err_i
 create_backup_dir(pgBackup *backup, const char *backup_instance_path)
 {
-	int    rc;
 	char   path[MAXPGPATH];
+	err_i  err;
 
 	join_path_components(path, backup_instance_path, base36enc(backup->backup_id));
 
 	/* TODO: add wrapper for remote mode */
-	rc = fio_mkdir(FIO_BACKUP_HOST, path, DIR_PERMISSION, true);
-
-	if (rc == 0)
+	err = $i(pioMakeDir, backup->backup_location, .path = path,
+			 .mode = DIR_PERMISSION, .strict = true);
+	if (!$haserr(err))
+	{
 		backup->root_dir = pgut_strdup(path);
-	else
-		elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno));
-	return rc;
+	} else {
+		elog(ERROR, "Can not create backup directory: %s", $errmsg(err));
+	}
+
+	return err;
 }
 
 /*
@@ -2969,6 +2979,9 @@ pgBackupInit(pgBackup *backup)
 	backup->files = NULL;
 	backup->note = NULL;
 	backup->content_crc = 0;
+
+	backup->backup_location = pioDriveForLocation(FIO_BACKUP_HOST);
+	backup->database_location = pioDriveForLocation(FIO_DB_HOST);
 }
 
 /* free pgBackup object */
@@ -2977,6 +2990,10 @@ pgBackupFree(void *backup)
 {
 	pgBackup *b = (pgBackup *) backup;
 
+	/* Both point to global static vars */
+	b->backup_location.self = NULL;
+	b->database_location.self = NULL;
+
 	pg_free(b->primary_conninfo);
 	pg_free(b->external_dir_str);
 	pg_free(b->root_dir);

src/catchup.c

@@ -613,6 +613,7 @@ int
 do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files,
 	parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list)
 {
+	pioDrive_i local_location = pioDriveForLocation(FIO_LOCAL_HOST);
 	PGconn		*source_conn = NULL;
 	PGNodeInfo	source_node_info;
 	bool		backup_logs = false;
@@ -632,6 +633,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 	ssize_t		transfered_datafiles_bytes = 0;
 	ssize_t		transfered_walfiles_bytes = 0;
 	char		pretty_source_bytes[20];
+	err_i		err = $noerr();
+
 
 	source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata);
 	catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata);
@@ -704,7 +707,12 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 	join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR);
 	if (!dry_run)
 	{
-		fio_mkdir(FIO_LOCAL_HOST, dest_xlog_path, DIR_PERMISSION, false);
+		err = $i(pioMakeDir, local_location, .path = dest_xlog_path,
+				 .mode = DIR_PERMISSION, .strict = false);
+		if($haserr(err))
+		{
+			elog(ERROR, "Can not create WAL directory: %s", $errmsg(err));
+		}
 		start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt,
 							current.start_lsn, current.tli, false);
 	}
@@ -820,7 +828,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 
 			elog(LOG, "Create directory '%s'", dirpath);
 			if (!dry_run)
-				fio_mkdir(FIO_LOCAL_HOST, dirpath, DIR_PERMISSION, false);
+			{
+				err = $i(pioMakeDir, local_location, .path = dirpath,
+						 .mode = DIR_PERMISSION, .strict = false);
+				if ($haserr(err))
+				{
+					elog(ERROR, "Can not create directory: %s", $errmsg(err));
+				}
+			}
 		}
 		else
 		{
@@ -854,9 +869,13 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
 			if (!dry_run)
 			{
 				/* create tablespace directory */
-				if (fio_mkdir(FIO_LOCAL_HOST, linked_path, file->mode, false) != 0)
-					elog(ERROR, "Could not create tablespace directory \"%s\": %s",
-						 linked_path, strerror(errno));
+				err = $i(pioMakeDir, local_location, .path = linked_path,
+						 .mode = file->mode, .strict = false);
+				if ($haserr(err))
+				{
+					elog(ERROR, "Could not create tablespace directory \"%s\": \"%s\"",
+						 linked_path, $errmsg(err));
+				}
 
 				/* create link to linked_path */
 				if (fio_symlink(FIO_LOCAL_HOST, linked_path, to_path, true) < 0)

src/dir.c

@@ -847,10 +847,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 						bool extract_tablespaces, bool incremental, fio_location location, 
 						const char* waldir_path)
 {
+	pioDrive_i drive = pioDriveForLocation(location);
 	int			i;
 	parray		*links = NULL;
 	mode_t		pg_tablespace_mode = DIR_PERMISSION;
 	char		to_path[MAXPGPATH];
+	err_i		err = $noerr();
+
 
 	if (waldir_path && !dir_is_empty(waldir_path, location))
 	{
@@ -932,7 +935,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 				waldir_path, to_path);
 
 			/* create tablespace directory from waldir_path*/
-			fio_mkdir(location, waldir_path, pg_tablespace_mode, false);
+			err = $i(pioMakeDir, drive, .path = waldir_path,
+					 .mode = pg_tablespace_mode, .strict = false);
+			if ($haserr(err))
+			{
+				elog(ERROR, "Can not create tablespace directory: %s",
+					 $errmsg(err));
+			}
 
 			/* create link to linked_path */
 			if (fio_symlink(location, waldir_path, to_path, incremental) < 0)
@@ -974,7 +983,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 							 linked_path, to_path);
 
 					/* create tablespace directory */
-					fio_mkdir(location, linked_path, pg_tablespace_mode, false);
+					err = $i(pioMakeDir, drive, .path = linked_path,
+							 .mode = pg_tablespace_mode, .strict = false);
+					if ($haserr(err))
+					{
+						elog(ERROR, "Can not create tablespace directory: %s",
+							 $errmsg(err));
+					}
 
 					/* create link to linked_path */
 					if (fio_symlink(location, linked_path, to_path, incremental) < 0)
@@ -991,8 +1006,13 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 
 		join_path_components(to_path, data_dir, dir->rel_path);
 
-		// TODO check exit code
-		fio_mkdir(location, to_path, dir->mode, false);
+		err = $i(pioMakeDir, drive, .path = to_path, .mode = dir->mode,
+				 .strict = false);
+		if ($haserr(err))
+		{
+			elog(ERROR, "Can not create tablespace directory: %s",
+				 $errmsg(err));
+		}
 	}
 
 	if (extract_tablespaces)

src/init.c

@@ -18,7 +18,9 @@
 int
 do_init(CatalogState *catalogState)
 {
+	pioDrive_i	backup_location = pioDriveForLocation(FIO_BACKUP_HOST);
 	int			results;
+	err_i		err;
 
 	results = pg_check_dir(catalogState->catalog_path);
 
@@ -32,13 +34,31 @@ do_init(CatalogState *catalogState)
 	}
 
 	/* create backup catalog root directory */
-	fio_mkdir(FIO_BACKUP_HOST, catalogState->catalog_path, DIR_PERMISSION, false);
+	err = $i(pioMakeDir, backup_location, .path = catalogState->catalog_path,
+			 .mode = DIR_PERMISSION, .strict = false);
+	if ($haserr(err))
+	{
+		elog(ERROR, "Can not create backup catalog root directory: %s",
+			 $errmsg(err));
+	}
 
 	/* create backup catalog data directory */
-	fio_mkdir(FIO_BACKUP_HOST, catalogState->backup_subdir_path, DIR_PERMISSION, false);
+	err = $i(pioMakeDir, backup_location, .path = catalogState->backup_subdir_path,
+			 .mode = DIR_PERMISSION, .strict = false);
+	if ($haserr(err))
+	{
+		elog(ERROR, "Can not create backup catalog data directory: %s",
+			 $errmsg(err));
+	}
 
 	/* create backup catalog wal directory */
-	fio_mkdir(FIO_BACKUP_HOST, catalogState->wal_subdir_path, DIR_PERMISSION, false);
+	err = $i(pioMakeDir, backup_location, .path = catalogState->wal_subdir_path,
+			 .mode = DIR_PERMISSION, .strict = false);
+	if ($haserr(err))
+	{
+		elog(ERROR, "Can not create backup catalog WAL directory: %s",
+			 $errmsg(err));
+	}
 
 	elog(INFO, "Backup catalog '%s' successfully inited", catalogState->catalog_path);
 	return 0;
@@ -47,8 +67,10 @@ do_init(CatalogState *catalogState)
 int
 do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
 {
+	pioDrive_i	backup_location = pioDriveForLocation(FIO_BACKUP_HOST);
 	struct stat st;
 	CatalogState *catalogState = instanceState->catalog_state;
+	err_i		err;
 
 	/* PGDATA is always required */
 	if (instance->pgdata == NULL)
@@ -85,8 +107,19 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
 				instanceState->instance_name, instanceState->instance_wal_subdir_path);
 
 	/* Create directory for data files of this specific instance */
-	fio_mkdir(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path, DIR_PERMISSION, false);
-	fio_mkdir(FIO_BACKUP_HOST, instanceState->instance_wal_subdir_path, DIR_PERMISSION, false);
+	err = $i(pioMakeDir, backup_location, .path = instanceState->instance_backup_subdir_path,
+			 .mode = DIR_PERMISSION, .strict = false);
+	if ($haserr(err))
+	{
+		elog(ERROR, "Can not create instance backup directory: %s",
+			 $errmsg(err));
+	}
+	err = $i(pioMakeDir, backup_location, .path = instanceState->instance_wal_subdir_path,
+			 .mode = DIR_PERMISSION, .strict = false);
+	if ($haserr(err))
+	{
+		elog(ERROR, "Can not create instance WAL directory: %s", $errmsg(err));
+	}
 
 	/*
 	 * Write initial configuration file.

src/merge.c

@@ -461,6 +461,8 @@ merge_chain(InstanceState *instanceState,
 	/* in-place merge flags */
 	bool		compression_match = false;
 	bool		program_version_match = false;
+	err_i		err = $noerr();
+
 	/* It's redundant to check block checksumms during merge */
 	skip_block_validation = true;
 
@@ -645,7 +647,13 @@ merge_chain(InstanceState *instanceState,
 			makeExternalDirPathByNum(new_container, full_external_prefix,
 									 file->external_dir_num);
 			join_path_components(dirpath, new_container, file->rel_path);
-			fio_mkdir(FIO_BACKUP_HOST, dirpath, DIR_PERMISSION, false);
+			err = $i(pioMakeDir, dest_backup->backup_location, .path = dirpath,
+					 .mode = DIR_PERMISSION, .strict = false);
+			if ($haserr(err))
+			{
+				elog(ERROR, "Can not create backup external directory: %s",
+					 $errmsg(err));
+			}
 		}
 
 		pg_atomic_init_flag(&file->lock);

src/pg_probackup.c

@@ -294,6 +294,7 @@ main(int argc, char *argv[])
 
 	ft_init_log(elog_ft_log);
 	fobj_init();
+	FOBJ_FUNC_ARP();
 	init_pio_objects();
 
 	PROGRAM_NAME_FULL = argv[0];

src/pg_probackup.h

@@ -488,6 +488,9 @@ struct pgBackup
 
 	/* map used for access to page headers */
 	HeaderMap       hdr_map;
+
+	pioDrive_i		database_location; /* Where to backup from/restore to */
+	pioDrive_i		backup_location; /* Where to save to/read from */
 };
 
 /* Recovery target for restore and validate subcommands */