rework server bootstrap completion

* replace server pids pthread mutex/cond with ABT versions
* add margo_state_dump() on client-server or server-server failures
  (currently commented out)
* add a 'bootstrap complete' broadcast rpc after rank 0 sees all
  servers have reported
* fix function declaration for unifyfs_invoke_broadcast_extents()

client/src/margo_client.c

@@ -224,14 +224,14 @@ static hg_handle_t create_handle(hg_id_t id)
  return handle;
 }
 
-static int forward_to_server(
- hg_handle_t hdl,
- void* input_ptr,
- double timeout_msec)
+static int forward_to_server(hg_handle_t hdl,
+ void* input_ptr,
+ double timeout_msec)
 {
  hg_return_t hret = margo_forward_timed(hdl, input_ptr, timeout_msec);
  if (hret != HG_SUCCESS) {
  LOGERR("margo_forward_timed() failed - %s", HG_Error_to_string(hret));
+ //margo_state_dump(client_rpc_context->mid, "-", 0, NULL);
  return UNIFYFS_ERROR_MARGO;
  }
  return UNIFYFS_SUCCESS;

common/src/unifyfs_server_rpcs.h

@@ -42,9 +42,9 @@ typedef enum {
  UNIFYFS_SERVER_RPC_LAMINATE,
  UNIFYFS_SERVER_RPC_METAGET,
  UNIFYFS_SERVER_RPC_METASET,
- UNIFYFS_SERVER_RPC_PID_REPORT,
  UNIFYFS_SERVER_RPC_TRANSFER,
  UNIFYFS_SERVER_RPC_TRUNCATE,
+ UNIFYFS_SERVER_BCAST_RPC_BOOTSTRAP,
  UNIFYFS_SERVER_BCAST_RPC_EXTENTS,
  UNIFYFS_SERVER_BCAST_RPC_FILEATTR,
  UNIFYFS_SERVER_BCAST_RPC_LAMINATE,
@@ -186,6 +186,13 @@ MERCURY_GEN_PROC(bcast_progress_out_t,
  ((int32_t)(ret)))
 DECLARE_MARGO_RPC_HANDLER(bcast_progress_rpc)
 
+/* Broadcast 'bootstrap complete' to all servers */
+MERCURY_GEN_PROC(bootstrap_complete_bcast_in_t,
+ ((int32_t)(root)))
+MERCURY_GEN_PROC(bootstrap_complete_bcast_out_t,
+ ((int32_t)(ret)))
+DECLARE_MARGO_RPC_HANDLER(bootstrap_complete_bcast_rpc)
+
 /* Broadcast file extents to all servers */
 MERCURY_GEN_PROC(extent_bcast_in_t,
  ((int32_t)(root))

server/src/margo_server.c

@@ -155,6 +155,11 @@ static void register_server_server_rpcs(margo_instance_id mid)
  bcast_progress_in_t, bcast_progress_out_t,
  bcast_progress_rpc);
 
+ unifyfsd_rpc_context->rpcs.bootstrap_complete_bcast_id =
+ MARGO_REGISTER(mid, "bootstrap_complete_bcast_rpc",
+ bootstrap_complete_bcast_in_t, bootstrap_complete_bcast_out_t,
+ bootstrap_complete_bcast_rpc);
+
  unifyfsd_rpc_context->rpcs.chunk_read_request_id =
  MARGO_REGISTER(mid, "chunk_read_request_rpc",
  chunk_read_request_in_t, chunk_read_request_out_t,
@@ -534,7 +539,7 @@ int margo_connect_servers(void)
 
  /* allocate array of structs to record address for each server */
  server_infos = (server_info_t*) calloc(glb_num_servers,
- sizeof(server_info_t));
+  sizeof(server_info_t));
  if (NULL == server_infos) {
  LOGERR("failed to allocate server_info array");
  return ENOMEM;

server/src/margo_server.h

@@ -32,6 +32,7 @@
 typedef struct ServerRpcIds {
  /* server-server rpcs */
  hg_id_t bcast_progress_id;
+ hg_id_t bootstrap_complete_bcast_id;
  hg_id_t chunk_read_request_id;
  hg_id_t chunk_read_response_id;
  hg_id_t extent_add_id;

server/src/unifyfs_global.h

@@ -208,8 +208,12 @@ bool check_pending_metaget(int gfid);
 unifyfs_rc clear_pending_metaget(int gfid);
 
 
-/* publish the pids of all servers to a shared file */
-int unifyfs_publish_server_pids(void);
+
+/* notify local server main thread that bootstrap is complete */
+int unifyfs_signal_bootstrap_complete(void);
+
+/* participate in collective server bootstrap completion process */
+int unifyfs_complete_bootstrap(void);
 
 /* report the pid for a server with given rank */
 int unifyfs_report_server_pid(int rank, int pid);