Skip to content

Commit 214f6a6

Browse files
hjelmnhjelmn
committed
btl/sm: remove emulated RDMA code
This removes the emulated RDMA code provided as a fallback so btl/sm could be used for osc/rdma. This functionality is now provided by ```btl_base_am_rdma_init ()``` so the dedicated version in btl/sm is no longer needed. Signed-off-by: Nathan Hjelm <hjelmn@google.com>
1 parent a2b5bde commit 214f6a6

File tree

7 files changed

+5
-245
lines changed

7 files changed

+5
-245
lines changed

‎opal/mca/btl/sm/Makefile.am‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ libmca_btl_sm_la_sources = \
4242
btl_sm_xpmem.h \
4343
btl_sm_knem.c \
4444
btl_sm_knem.h \
45-
btl_sm_sc_emu.c \
4645
btl_sm_atomic.c
4746

4847
# Make the output library in this directory, and name it either

‎opal/mca/btl/sm/btl_sm.h‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,6 @@ enum {
103103
MCA_BTL_SM_CMA = 1,
104104
MCA_BTL_SM_KNEM = 2,
105105
MCA_BTL_SM_NONE = 3,
106-
MCA_BTL_SM_EMUL = 4,
107106
};
108107

109108
/**
@@ -246,11 +245,6 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
246245
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
247246
#endif
248247

249-
int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
250-
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
251-
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
252-
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
253-
254248
/**
255249
* Initiate an synchronous get.
256250
*
@@ -281,29 +275,6 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
281275

282276
ino_t mca_btl_sm_get_user_ns_id(void);
283277

284-
int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
285-
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
286-
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
287-
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
288-
289-
int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
290-
uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
291-
mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
292-
mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
293-
294-
int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
295-
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
296-
mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
297-
uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
298-
void *cbcontext, void *cbdata);
299-
300-
int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
301-
void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
302-
mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
303-
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
304-
305-
void mca_btl_sm_sc_emu_init (void);
306-
307278
/**
308279
* Allocate a segment.
309280
*

‎opal/mca/btl/sm/btl_sm_component.c‎

Lines changed: 1 addition & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ static mca_base_var_enum_value_t single_copy_mechanisms[] = {
7777
#if OPAL_BTL_SM_HAVE_KNEM
7878
{.value = MCA_BTL_SM_KNEM, .string = "knem"},
7979
#endif
80-
{.value = MCA_BTL_SM_EMUL, .string = "emulated"},
8180
{.value = MCA_BTL_SM_NONE, .string = "none"},
8281
{.value = 0, .string = NULL}
8382
};
@@ -254,28 +253,7 @@ static int mca_btl_sm_component_register (void)
254253
mca_btl_sm.super.btl_rdma_pipeline_send_length = mca_btl_sm.super.btl_eager_limit;
255254
mca_btl_sm.super.btl_rdma_pipeline_frag_size = mca_btl_sm.super.btl_eager_limit;
256255

257-
#if OPAL_HAVE_ATOMIC_MATH_64
258-
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA |
259-
MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS;
260-
261-
mca_btl_sm.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND |
262-
MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
263-
MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_SWAP;
264-
#if OPAL_HAVE_ATOMIC_MATH_32
265-
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_32BIT;
266-
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
267-
268-
#if OPAL_HAVE_ATOMIC_MIN_64
269-
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN;
270-
#endif /* OPAL_HAVE_ATOMIC_MIN_64 */
271-
272-
#if OPAL_HAVE_ATOMIC_MAX_64
273-
mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MAX;
274-
#endif /* OPAL_HAVE_ATOMIC_MAX_64 */
275-
276-
#else
277-
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA;
278-
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
256+
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
279257

280258
if (MCA_BTL_SM_NONE != mca_btl_sm_component.single_copy_mechanism) {
281259
/* True single copy mechanisms should provide better bandwidth */
@@ -284,12 +262,6 @@ static int mca_btl_sm_component_register (void)
284262
mca_btl_sm.super.btl_bandwidth = 10000; /* Mbs */
285263
}
286264

287-
mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
288-
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
289-
mca_btl_sm.super.btl_atomic_op = mca_btl_sm_emu_aop;
290-
mca_btl_sm.super.btl_atomic_fop = mca_btl_sm_emu_afop;
291-
mca_btl_sm.super.btl_atomic_cswap = mca_btl_sm_emu_acswap;
292-
293265
mca_btl_sm.super.btl_latency = 1; /* Microsecs */
294266

295267
/* Call the BTL based to register its MCA params */
@@ -427,9 +399,6 @@ static void mca_btl_sm_check_single_copy (void)
427399
int initial_mechanism = mca_btl_sm_component.single_copy_mechanism;
428400
#endif
429401

430-
/* single-copy emulation is always used to support AMO's right now */
431-
mca_btl_sm_sc_emu_init ();
432-
433402
#if OPAL_BTL_SM_HAVE_XPMEM
434403
if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
435404
/* try to create an xpmem segment for the entire address space */

‎opal/mca/btl/sm/btl_sm_get.c‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -164,16 +164,3 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
164164
return OPAL_SUCCESS;
165165
}
166166
#endif
167-
168-
int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
169-
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
170-
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
171-
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
172-
{
173-
if (size > mca_btl_sm.super.btl_get_limit) {
174-
return OPAL_ERR_NOT_AVAILABLE;
175-
}
176-
177-
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_GET, 0, 0, 0, order, flags, size,
178-
local_address, remote_address, cbfunc, cbcontext, cbdata);
179-
}

‎opal/mca/btl/sm/btl_sm_module.c‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
* and Technology (RIST). All rights reserved.
2020
* Copyright (c) 2018-2019 Triad National Security, LLC. All rights
2121
* reserved.
22-
* Copyright (c) 2020 Google, LLC. All rights reserved.
22+
* Copyright (c) 2020-2021 Google, LLC. All rights reserved.
2323
* Copyright (c) 2021 Nanook Consulting. All rights reserved.
2424
* $COPYRIGHT$
2525
*
@@ -266,8 +266,6 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
266266
* Currently the kernel does not allow * process_vm_{read,write}v()
267267
* for processes running in different user namespaces even if
268268
* all involved user IDs are mapped to the same user ID.
269-
*
270-
* Fallback to MCA_BTL_SM_EMUL.
271269
*/
272270
if (MCA_BASE_VAR_SOURCE_DEFAULT != source) {
273271
/* If CMA has been explicitly selected we want to error out */
@@ -281,11 +279,9 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
281279
*/
282280
opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-warning",
283281
true, opal_process_info.nodename);
284-
mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_EMUL;
285-
mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
286-
mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
287-
mca_btl_sm.super.btl_put_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
288-
mca_btl_sm.super.btl_get_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
282+
mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_NONE;
283+
mca_btl_sm.super.btl_get = NULL;
284+
mca_btl_sm.super.btl_put = NULL;
289285
}
290286
}
291287
#if OPAL_BTL_SM_HAVE_XPMEM

‎opal/mca/btl/sm/btl_sm_put.c‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -143,19 +143,3 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
143143
return OPAL_SUCCESS;
144144
}
145145
#endif
146-
147-
/**
148-
* @brief Provides an emulated put path which uses copy-in copy-out with shared memory buffers
149-
*/
150-
int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
151-
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
152-
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
153-
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
154-
{
155-
if (size > mca_btl_sm.super.btl_put_limit) {
156-
return OPAL_ERR_NOT_AVAILABLE;
157-
}
158-
159-
return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_PUT, 0, 0, 0, order, flags, size,
160-
local_address, remote_address, cbfunc, cbcontext, cbdata);
161-
}

‎opal/mca/btl/sm/btl_sm_sc_emu.c‎

Lines changed: 0 additions & 146 deletions
This file was deleted.

0 commit comments

Comments
 (0)