Skip to content

Commit

Permalink
btl/sm: remove emulated RDMA code
Browse files Browse the repository at this point in the history 
This removes the emulated RDMA code provided as a fallback so btl/sm
could be used for osc/rdma. This functionality is now provided by
```btl_base_am_rdma_init ()``` so the dedicated version in btl/sm
is no longer needed.

Signed-off-by: Nathan Hjelm <hjelmn@google.com>
  • Loading branch information
@hjelmn
hjelmn committed Mar 2, 2021
1 parent 1cc6b78 commit 1fa152d
Show file tree
Hide file tree
Showing 7 changed files with 5 additions and 245 deletions.
1 change: 0 additions & 1 deletion opal/mca/btl/sm/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ libmca_btl_sm_la_sources = \
btl_sm_xpmem.h \
btl_sm_knem.c \
btl_sm_knem.h \
btl_sm_sc_emu.c \
btl_sm_atomic.c

# Make the output library in this directory, and name it either
Expand Down
29 changes: 0 additions & 29 deletions opal/mca/btl/sm/btl_sm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ enum {
MCA_BTL_SM_CMA = 1,
MCA_BTL_SM_KNEM = 2,
MCA_BTL_SM_NONE = 3,
MCA_BTL_SM_EMUL = 4,
};

/**
Expand Down Expand Up @@ -246,11 +245,6 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
#endif

int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);

/**
* Initiate an synchronous get.
*
Expand Down Expand Up @@ -281,29 +275,6 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en

ino_t mca_btl_sm_get_user_ns_id(void);

int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);

int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);

int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
void *cbcontext, void *cbdata);

int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);

void mca_btl_sm_sc_emu_init (void);

/**
* Allocate a segment.
*
Expand Down
33 changes: 1 addition & 32 deletions opal/mca/btl/sm/btl_sm_component.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ static mca_base_var_enum_value_t single_copy_mechanisms[] = {
#if OPAL_BTL_SM_HAVE_KNEM
{.value = MCA_BTL_SM_KNEM, .string = "knem"},
#endif
{.value = MCA_BTL_SM_EMUL, .string = "emulated"},
{.value = MCA_BTL_SM_NONE, .string = "none"},
{.value = 0, .string = NULL}
};
Expand Down Expand Up @@ -254,28 +253,7 @@ static int mca_btl_sm_component_register (void)
mca_btl_sm.super.btl_rdma_pipeline_send_length = mca_btl_sm.super.btl_eager_limit;
mca_btl_sm.super.btl_rdma_pipeline_frag_size = mca_btl_sm.super.btl_eager_limit;

#if OPAL_HAVE_ATOMIC_MATH_64
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA |
MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS;

mca_btl_sm.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND |
MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_SWAP;
#if OPAL_HAVE_ATOMIC_MATH_32
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_32BIT;
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */

#if OPAL_HAVE_ATOMIC_MIN_64
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN;
#endif /* OPAL_HAVE_ATOMIC_MIN_64 */

#if OPAL_HAVE_ATOMIC_MAX_64
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MAX;
#endif /* OPAL_HAVE_ATOMIC_MAX_64 */

#else
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA;
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;

if (MCA_BTL_SM_NONE != mca_btl_sm_component.single_copy_mechanism) {
/* True single copy mechanisms should provide better bandwidth */
Expand All @@ -284,12 +262,6 @@ static int mca_btl_sm_component_register (void)
mca_btl_sm.super.btl_bandwidth = 10000; /* Mbs */
}

mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
mca_btl_sm.super.btl_atomic_op = mca_btl_sm_emu_aop;
mca_btl_sm.super.btl_atomic_fop = mca_btl_sm_emu_afop;
mca_btl_sm.super.btl_atomic_cswap = mca_btl_sm_emu_acswap;

mca_btl_sm.super.btl_latency = 1; /* Microsecs */

/* Call the BTL based to register its MCA params */
Expand Down Expand Up @@ -427,9 +399,6 @@ static void mca_btl_sm_check_single_copy (void)
int initial_mechanism = mca_btl_sm_component.single_copy_mechanism;
#endif

/* single-copy emulation is always used to support AMO's right now */
mca_btl_sm_sc_emu_init ();

#if OPAL_BTL_SM_HAVE_XPMEM
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
/* try to create an xpmem segment for the entire address space */
Expand Down
13 changes: 0 additions & 13 deletions opal/mca/btl/sm/btl_sm_get.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,16 +167,3 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
return OPAL_SUCCESS;
}
#endif

int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{
if (size > mca_btl_sm.super.btl_get_limit) {
return OPAL_ERR_NOT_AVAILABLE;
}

return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_GET, 0, 0, 0, order, flags, size,
local_address, remote_address, cbfunc, cbcontext, cbdata);
}
12 changes: 4 additions & 8 deletions opal/mca/btl/sm/btl_sm_module.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
* and Technology (RIST). All rights reserved.
* Copyright (c) 2018-2019 Triad National Security, LLC. All rights
* reserved.
* Copyright (c) 2020 Google, LLC. All rights reserved.
* Copyright (c) 2020-2021 Google, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -265,8 +265,6 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
* Currently the kernel does not allow * process_vm_{read,write}v()
* for processes running in different user namespaces even if
* all involved user IDs are mapped to the same user ID.
*
* Fallback to MCA_BTL_SM_EMUL.
*/
if (MCA_BASE_VAR_SOURCE_DEFAULT != source) {
/* If CMA has been explicitly selected we want to error out */
Expand All @@ -280,11 +278,9 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
*/
opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-warning",
true, opal_process_info.nodename);
mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_EMUL;
mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
mca_btl_sm.super.btl_put_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
mca_btl_sm.super.btl_get_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_NONE;
mca_btl_sm.super.btl_get = NULL;
mca_btl_sm.super.btl_put = NULL;
}
}
#if OPAL_BTL_SM_HAVE_XPMEM
Expand Down
16 changes: 0 additions & 16 deletions opal/mca/btl/sm/btl_sm_put.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,19 +143,3 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
return OPAL_SUCCESS;
}
#endif

/**
* @brief Provides an emulated put path which uses copy-in copy-out with shared memory buffers
*/
int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{
if (size > mca_btl_sm.super.btl_put_limit) {
return OPAL_ERR_NOT_AVAILABLE;
}

return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_PUT, 0, 0, 0, order, flags, size,
local_address, remote_address, cbfunc, cbcontext, cbdata);
}
146 changes: 0 additions & 146 deletions opal/mca/btl/sm/btl_sm_sc_emu.c
View file

This file was deleted.

0 comments on commit 1fa152d

Please sign in to comment.