Define monitor unsubscribe_on_delete

Makefile.am

@@ -92,6 +92,8 @@ common_srcs =				\
 	prov/util/src/ze_ipc_monitor.c	\
 	prov/util/src/xpmem_monitor.c	\
         prov/util/src/kdreg2_mem_monitor.c \
+	prov/util/src/uffd_mem_monitor.c \
+	prov/util/src/import_mem_monitor.c \
 	prov/util/src/util_profile.c \
 	prov/coll/src/coll_attr.c	\
 	prov/coll/src/coll_av.c		\

include/ofi_mr.h

@@ -118,9 +118,12 @@ static inline uint64_t ofi_mr_get_prov_mode(uint32_t version,
 	}
 }
 
-
 /* Single lock used by all memory monitors and MR caches. */
 extern pthread_mutex_t mm_lock;
+
+/* Lock used to coordinate monitor states. */
+extern pthread_mutex_t mm_state_lock;
+
 /* The read-write lock is an additional lock used to protect the dlist_entry
  * list of ofi_mem_monitor. Due to the necessity of releasing the mm_lock
  * while walking the dlist in ofi_monitor_notify, we need a separate lock to
@@ -189,6 +192,17 @@ struct ofi_mem_monitor {
 	bool (*valid)(struct ofi_mem_monitor *notifier,
 		      const struct ofi_mr_info *info, struct ofi_mr_entry *entry);
 	const char *name;
+
+	/* Set this to true if the monitor tracks each individual memory
+	 * registration.  That is, the monitor has a 1-to-1 mapping to MR Cache
+	 * entry and corresponding subscribe.
+	 *
+	 * When true, the MR Cache will call the memory monitor unsubscribe()
+	 * method when deleting an MR Cache entry.  This allows the monitor
+	 * stop tracking memory regions for which there are no longer
+	 * registrations.
+	 */
+	bool unsubscribe_on_delete;
 };
 
 void ofi_monitor_init(struct ofi_mem_monitor *monitor);

libfabric.vcxproj

@@ -760,6 +760,8 @@
     <ClCompile Include="prov\util\src\rocr_ipc_monitor.c" />
     <ClCompile Include="prov\util\src\xpmem_monitor.c" />
     <ClCompile Include="prov\util\src\kdreg2_mem_monitor.c" />
+    <ClCompile Include="prov\util\src\uffd_mem_monitor.c" />
+    <ClCompile Include="prov\util\src\import_mem_monitor.c" />
     <ClCompile Include="prov\coll\src\coll_attr.c" />
     <ClCompile Include="prov\coll\src\coll_av.c" />
     <ClCompile Include="prov\coll\src\coll_av_set.c" />

prov/util/src/cuda_ipc_monitor.c

@@ -64,6 +64,7 @@ static struct ofi_mem_monitor cuda_ipc_monitor_ = {
 	.unsubscribe = ofi_monitor_unsubscribe_no_op,
 	.valid = cuda_ipc_monitor_valid,
 	.name = "cuda_ipc",
+	.unsubscribe_on_delete = false,
 };
 
 struct ofi_mem_monitor *cuda_ipc_monitor = &cuda_ipc_monitor_;

prov/util/src/cuda_mem_monitor.c

@@ -154,6 +154,7 @@ static struct ofi_mem_monitor cuda_mm = {
 	.unsubscribe = cuda_mm_unsubscribe,
 	.valid = cuda_mm_valid,
 	.name = "cuda",
+	.unsubscribe_on_delete = false,
 };
 
 struct ofi_mem_monitor *cuda_monitor = &cuda_mm;

prov/util/src/import_mem_monitor.c

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2017 Cray Inc. All rights reserved.
+ * Copyright (c) 2017-2021 Intel Inc. All rights reserved.
+ * Copyright (c) 2019-2021 Amazon.com, Inc. or its affiliates.
+ *                         All rights reserved.
+ * (C) Copyright 2020,2024 Hewlett Packard Enterprise Development LP
+ * Copyright (C) 2024 Cornelis Networks. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <unistd.h>
+
+#include <ofi_mr.h>
+#include <ofi_enosys.h>
+#include <rdma/fi_ext.h>
+
+static void ofi_import_monitor_init(struct ofi_mem_monitor *monitor);
+static void ofi_import_monitor_cleanup(struct ofi_mem_monitor *monitor);
+static int ofi_import_monitor_start(struct ofi_mem_monitor *monitor);
+static void ofi_import_monitor_stop(struct ofi_mem_monitor *monitor);
+static int ofi_import_monitor_subscribe(struct ofi_mem_monitor *notifier,
+					const void *addr, size_t len,
+					union ofi_mr_hmem_info *hmem_info);
+static void ofi_import_monitor_unsubscribe(struct ofi_mem_monitor *notifier,
+					   const void *addr, size_t len,
+					   union ofi_mr_hmem_info *hmem_info);
+static bool ofi_import_monitor_valid(struct ofi_mem_monitor *notifier,
+				     const struct ofi_mr_info *info,
+				     struct ofi_mr_entry *entry);
+
+struct ofi_import_monitor {
+	struct ofi_mem_monitor monitor;
+	struct fid_mem_monitor *impfid;
+};
+
+static struct ofi_import_monitor impmon = {
+	.monitor.iface = FI_HMEM_SYSTEM,
+	.monitor.init = ofi_import_monitor_init,
+	.monitor.cleanup = ofi_import_monitor_cleanup,
+	.monitor.start = ofi_import_monitor_start,
+	.monitor.stop = ofi_import_monitor_stop,
+	.monitor.subscribe = ofi_import_monitor_subscribe,
+	.monitor.unsubscribe = ofi_import_monitor_unsubscribe,
+	.monitor.valid = ofi_import_monitor_valid,
+	.monitor.name = "import",
+	.monitor.unsubscribe_on_delete = false,
+};
+
+struct ofi_mem_monitor *import_monitor = &impmon.monitor;
+
+static void ofi_import_monitor_init(struct ofi_mem_monitor *monitor)
+{
+	ofi_monitor_init(monitor);
+}
+
+static void ofi_import_monitor_cleanup(struct ofi_mem_monitor *monitor)
+{
+	assert(!impmon.impfid);
+	ofi_monitor_cleanup(monitor);
+}
+
+static int ofi_import_monitor_start(struct ofi_mem_monitor *monitor)
+{
+	if (!impmon.impfid)
+		return -FI_ENOSYS;
+
+	return impmon.impfid->export_ops->start(impmon.impfid);
+}
+
+static void ofi_import_monitor_stop(struct ofi_mem_monitor *monitor)
+{
+	assert(impmon.impfid);
+	impmon.impfid->export_ops->stop(impmon.impfid);
+}
+
+static int ofi_import_monitor_subscribe(struct ofi_mem_monitor *notifier,
+					const void *addr, size_t len,
+					union ofi_mr_hmem_info *hmem_info)
+{
+	assert(impmon.impfid);
+	return impmon.impfid->export_ops->subscribe(impmon.impfid, addr, len);
+}
+
+static void ofi_import_monitor_unsubscribe(struct ofi_mem_monitor *notifier,
+					   const void *addr, size_t len,
+					   union ofi_mr_hmem_info *hmem_info)
+{
+	assert(impmon.impfid);
+	impmon.impfid->export_ops->unsubscribe(impmon.impfid, addr, len);
+}
+
+static bool ofi_import_monitor_valid(struct ofi_mem_monitor *notifier,
+				     const struct ofi_mr_info *info,
+				     struct ofi_mr_entry *entry)
+{
+	assert(impmon.impfid);
+	return impmon.impfid->export_ops->valid(impmon.impfid,
+						entry->info.iov.iov_base,
+						entry->info.iov.iov_len);
+}
+
+static void ofi_import_monitor_notify(struct fid_mem_monitor *monitor,
+				      const void *addr, size_t len)
+{
+	assert(monitor->fid.context == &impmon);
+	pthread_rwlock_rdlock(&mm_list_rwlock);
+	pthread_mutex_lock(&mm_lock);
+	ofi_monitor_notify(&impmon.monitor, addr, len);
+	pthread_mutex_unlock(&mm_lock);
+	pthread_rwlock_unlock(&mm_list_rwlock);
+}
+
+static int ofi_close_import(struct fid *fid)
+{
+	pthread_mutex_lock(&mm_state_lock);
+	impmon.monitor.state = FI_MM_STATE_IDLE;
+	pthread_mutex_unlock(&mm_state_lock);
+	impmon.impfid = NULL;
+	return 0;
+}
+
+static struct fi_ops_mem_notify import_ops = {
+	.size = sizeof(struct fi_ops_mem_notify),
+	.notify = ofi_import_monitor_notify,
+};
+
+static struct fi_ops impfid_ops = {
+	.size = sizeof(struct fi_ops),
+	.close = ofi_close_import,
+	.bind = fi_no_bind,
+	.control = fi_no_control,
+	.ops_open = fi_no_ops_open,
+	.tostr = fi_no_tostr,
+	.ops_set = fi_no_ops_set,
+};
+
+int ofi_monitor_import(struct fid *fid)
+{
+	struct fid_mem_monitor *impfid;
+
+	if (fid->fclass != FI_CLASS_MEM_MONITOR)
+		return -FI_ENOSYS;
+
+	if (impmon.impfid) {
+		FI_WARN(&core_prov, FI_LOG_MR,
+			"imported monitor already exists\n");
+		return -FI_EBUSY;
+	}
+
+	if (default_monitor && !dlist_empty(&default_monitor->list)) {
+		FI_WARN(&core_prov, FI_LOG_MR,
+			"cannot replace active monitor\n");
+		return -FI_EBUSY;
+	}
+
+	impfid = container_of(fid, struct fid_mem_monitor, fid);
+	if (impfid->export_ops->size < sizeof(struct fi_ops_mem_monitor))
+		return -FI_EINVAL;
+
+	impmon.impfid = impfid;
+	impfid->fid.context = &impmon;
+	impfid->fid.ops = &impfid_ops;
+	impfid->import_ops = &import_ops;
+
+	FI_INFO(&core_prov, FI_LOG_MR,
+		"setting imported memory monitor as default\n");
+	default_monitor = &impmon.monitor;
+	return 0;
+}

prov/util/src/kdreg2_mem_monitor.c

@@ -39,6 +39,8 @@
 #define EVICTOR_THREAD_ATTR NULL
 #define INFINITE_TIMEOUT -1
 
+/* kdreg2 monitors each address range (byte accurate) with a unique cookie */
+
 static int kdreg2_monitor_subscribe(struct ofi_mem_monitor *monitor,
 				    const void *addr, size_t len,
 				    union ofi_mr_hmem_info *hmem_info)
@@ -63,6 +65,8 @@ static int kdreg2_monitor_subscribe(struct ofi_mem_monitor *monitor,
 	return 0;
 }
 
+/* Unsubscribe is via cookie, not address range */
+
 static void kdreg2_monitor_unsubscribe(struct ofi_mem_monitor *monitor,
 				       const void *addr, size_t len,
 				       union ofi_mr_hmem_info *hmem_info)
@@ -359,6 +363,7 @@ static struct ofi_kdreg2 kdreg2_mm = {
 	.monitor.unsubscribe = kdreg2_monitor_unsubscribe,
 	.monitor.valid       = kdreg2_monitor_valid,
 	.monitor.name        = "kdreg2",
+	.monitor.unsubscribe_on_delete = true,
 	.fd                  = -1,
 	.exit_pipe           = { -1, -1 },
 	.status_data         = NULL,

prov/util/src/rocr_ipc_monitor.c

@@ -65,6 +65,7 @@ static struct ofi_mem_monitor rocr_ipc_monitor_ = {
 	.unsubscribe = ofi_monitor_unsubscribe_no_op,
 	.valid = rocr_ipc_monitor_valid,
 	.name = "rocr_ipc",
+	.unsubscribe_on_delete = false,
 };
 
 struct ofi_mem_monitor *rocr_ipc_monitor = &rocr_ipc_monitor_;

prov/util/src/rocr_mem_monitor.c

@@ -73,6 +73,7 @@ static struct rocr_mm rocr_mm = {
 		.unsubscribe = rocr_mm_unsubscribe,
 		.valid = rocr_mm_valid,
 		.name = "rocr",
+		.unsubscribe_on_delete = false,
 	},
 };
 
@@ -404,6 +405,7 @@ static struct ofi_mem_monitor rocr_mm = {
 	.unsubscribe = rocr_mm_unsubscribe,
 	.valid = rocr_mm_valid,
 	.name = "rocr",
+	.unsubscribe_on_delete = false,
 };
 
 struct ofi_mem_monitor *rocr_monitor = &rocr_mm;