core/util: Adding MR cache malloc hooking mechanism

Integrate hooking memory allocation related calls into the MR cache. This allows us to support older linux kernel that do not support userfaultfd. The core of the code is adapted from OpenMPI. Signed-off-by: Mikhail Khalilov <mikhail.khalilov@intel.com> Signed-off-by: Nikita Gusev <nikita.gusev@intel.com> Signed-off-by: Sean Hefty <sean.hefty@intel.com>
shefty · Jul 30, 2019 · 84864d0 · 84864d0
1 parent 90c4198
commit 84864d0
Show file tree

Hide file tree

Showing 8 changed files with 615 additions and 10 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -66,6 +66,7 @@ common_srcs =				\
 	prov/util/src/util_ns.c		\
 	prov/util/src/util_shm.c	\
 	prov/util/src/util_mem_monitor.c\
+	prov/util/src/util_mem_hooks.c	\
 	prov/util/src/util_mr_cache.c
 
 

diff --git a/configure.ac b/configure.ac
@@ -451,6 +451,9 @@ AS_IF([test $have_uffd -eq 1],
 AC_DEFINE_UNQUOTED([HAVE_UFFD_UNMAP], [$have_uffd],
 	[Define to 1 if platform supports userfault fd unmap])
 
+dnl Check support to intercept syscalls
+AC_CHECK_HEADERS_ONCE(elf.h sys/auxv.h)
+
 dnl Provider-specific checks
 FI_PROVIDER_INIT
 FI_PROVIDER_SETUP([psm])

diff --git a/include/ofi_mr.h b/include/ofi_mr.h
@@ -46,7 +46,6 @@
 #include <ofi_list.h>
 #include <ofi_tree.h>
 
-
 struct ofi_mr_info {
 	struct iovec iov;
 };
@@ -125,6 +124,8 @@ int ofi_monitor_subscribe(struct ofi_mem_monitor *monitor,
 void ofi_monitor_unsubscribe(struct ofi_mem_monitor *monitor,
 			     const void *addr, size_t len);
 
+extern struct ofi_mem_monitor *default_monitor;
+
 /*
  * Userfault fd memory monitor
  */
@@ -139,6 +140,19 @@ void ofi_uffd_cleanup(void);
 
 extern struct ofi_mem_monitor *uffd_monitor;
 
+/*
+ * Memory intercept call memory monitor
+ */
+struct ofi_memhooks {
+	struct ofi_mem_monitor          monitor;
+	struct dlist_entry		intercept_list;
+};
+
+int ofi_memhooks_init(void);
+void ofi_memhooks_cleanup(void);
+
+extern struct ofi_mem_monitor *memhooks_monitor;
+
 
 /*
  * Used to store registered memory regions into a lookup map.  This
@@ -184,7 +198,7 @@ int ofi_mr_close(struct fid *fid);
 int ofi_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr,
 		   uint64_t flags, struct fid_mr **mr_fid);
 int ofi_mr_regv(struct fid *fid, const struct iovec *iov,
-	        size_t count, uint64_t access, uint64_t offset,
+		size_t count, uint64_t access, uint64_t offset,
 		uint64_t requested_key, uint64_t flags,
 		struct fid_mr **mr_fid, void *context);
 int ofi_mr_reg(struct fid *fid, const void *buf, size_t len,
@@ -201,6 +215,7 @@ struct ofi_mr_cache_params {
 	size_t				max_cnt;
 	size_t				max_size;
 	int				merge_regions;
+	char *				monitor;
 };
 
 extern struct ofi_mr_cache_params	cache_params;

diff --git a/libfabric.vcxproj b/libfabric.vcxproj
@@ -557,6 +557,7 @@
     <ClCompile Include="prov\util\src\util_poll.c" />
     <ClCompile Include="prov\util\src\util_wait.c" />
     <ClCompile Include="prov\util\src\util_mem_monitor.c" />
+    <ClCompile Include="prov\util\src\util_mem_hooks.c" />
     <ClCompile Include="prov\util\src\util_mr_cache.c" />
     <ClCompile Include="src\common.c" />
     <ClCompile Include="src\enosys.c">

diff --git a/libfabric.vcxproj.filters b/libfabric.vcxproj.filters
@@ -180,6 +180,9 @@
     <ClCompile Include="prov\util\src\util_mem_monitor.c">
       <Filter>Source Files\prov\util</Filter>
     </ClCompile>
+    <ClCompile Include="prov\util\src\util_mem_hooks.c">
+      <Filter>Source Files\prov\util</Filter>
+    </ClCompile>
     <ClCompile Include="prov\util\src\util_mr_cache.c">
       <Filter>Source Files\prov\util</Filter>
     </ClCompile>